diff --git a/PackagesList.cmake b/PackagesList.cmake index 1a0f15b870c0..70dd6441a763 100644 --- a/PackagesList.cmake +++ b/PackagesList.cmake @@ -102,7 +102,6 @@ TRIBITS_REPOSITORY_DEFINE_PACKAGES( ShyLU_Node packages/shylu/shylu_node PT Amesos2 packages/amesos2 PT SEACAS packages/seacas PT # Depends on netcdf, optionally hdf5, xdmf, pamgen - Trios packages/trios EX #temporary Komplex packages/komplex ST Anasazi packages/anasazi PT Ifpack2 packages/ifpack2 PT @@ -138,7 +137,6 @@ TRIBITS_REPOSITORY_DEFINE_PACKAGES( WebTrilinos packages/WebTrilinos EX # Should be ST NewPackage packages/new_package EX # Should be ST Optika packages/optika EX - Mesquite packages/mesquite ST TrilinosCouplings packages/trilinoscouplings PT Pike packages/pike PT xSDKTrilinos packages/xSDKTrilinos ST @@ -192,6 +190,4 @@ TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Anasazi Windows) TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Isorropia Windows) TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Zoltan Windows) TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Teko Windows) -TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Mesquite AIX) -TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Trios Windows) TRIBITS_DISABLE_PACKAGE_ON_PLATFORMS(Panzer Windows) diff --git a/cmake/ctest/drivers/trilinos-test2/nightly_create_tarball.py b/cmake/ctest/drivers/trilinos-test2/nightly_create_tarball.py index 3e50409db0d0..847513068395 100755 --- a/cmake/ctest/drivers/trilinos-test2/nightly_create_tarball.py +++ b/cmake/ctest/drivers/trilinos-test2/nightly_create_tarball.py @@ -123,7 +123,6 @@ ("TriKota", False), ("TrilinosCouplings", True), ("TrilinosFramework", False), - ("Trios", False), ("Triutils", True), ("WebTrilinos", False), ("Xpetra", True), diff --git a/cmake/dependencies/CDashSubprojectDependencies.xml b/cmake/dependencies/CDashSubprojectDependencies.xml index de482166f137..17a479b744c0 100644 --- a/cmake/dependencies/CDashSubprojectDependencies.xml +++ b/cmake/dependencies/CDashSubprojectDependencies.xml @@ -189,11 +189,6 @@ - - - - - diff --git a/cmake/dependencies/TrilinosPackageDependencies.xml b/cmake/dependencies/TrilinosPackageDependencies.xml index 96eb0377d4bf..655c49de8da8 100644 --- a/cmake/dependencies/TrilinosPackageDependencies.xml +++ b/cmake/dependencies/TrilinosPackageDependencies.xml @@ -1090,132 +1090,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cmake/std/atdm/ATDMDisables.cmake b/cmake/std/atdm/ATDMDisables.cmake index d825544981c8..017aeff74fbc 100644 --- a/cmake/std/atdm/ATDMDisables.cmake +++ b/cmake/std/atdm/ATDMDisables.cmake @@ -34,7 +34,6 @@ SET(ATDM_SE_PACKAGE_DISABLES Domi Pliris Komplex - Trios FEI TriKota STKClassic diff --git a/commonTools/git/hooks/Trilinos/dirs_to_emails b/commonTools/git/hooks/Trilinos/dirs_to_emails index 66f933f3cfb2..3feb39b417eb 100644 --- a/commonTools/git/hooks/Trilinos/dirs_to_emails +++ b/commonTools/git/hooks/Trilinos/dirs_to_emails @@ -60,7 +60,6 @@ packages/phdmesh phdmesh-checkins@software.sandia.gov packages/intrepid intrepid-checkins@software.sandia.gov packages/shards shards-checkins@software.sandia.gov packages/stk stk-checkins@software.sandia.gov -packages/trios trios-checkins@software.sandia.gov packages/seacas seacas-checkins@software.sandia.gov packages/teko teko-checkins@software.sandia.gov packages/muelu muelu-checkins@software.sandia.gov diff --git a/commonTools/release/tarball-do-configure b/commonTools/release/tarball-do-configure index d3c00b73d778..3bcd83243f36 100755 --- a/commonTools/release/tarball-do-configure +++ b/commonTools/release/tarball-do-configure @@ -89,7 +89,6 @@ cmake \ -DTrilinos_ENABLE_Tpetra=ON \ -DTrilinos_ENABLE_TriKota=ON \ -DTrilinos_ENABLE_TrilinosCouplings=ON \ --DTrilinos_ENABLE_Trios=ON \ -DTrilinos_ENABLE_Triutils=ON \ -DTrilinos_ENABLE_Zoltan=ON \ -DTrilinos_ENABLE_Zoltan2=ON \ diff --git a/packages/panzer/maintenance/build_panzer_operandi b/packages/panzer/maintenance/build_panzer_operandi index f15f4b62ec0c..4758e24535ff 100755 --- a/packages/panzer/maintenance/build_panzer_operandi +++ b/packages/panzer/maintenance/build_panzer_operandi @@ -26,7 +26,6 @@ cmake \ -D Panzer_STK_ENABLE_TESTS:BOOL=ON \ -D Panzer_STK_ENABLE_EXAMPLES:BOOL=ON \ -D Intrepid2_ENABLE_DEBUG_INF_CHECK=OFF \ --D Trios_ENABLE_XDMF:BOOL=OFF \ -D TPL_ENABLE_MPI:BOOL=ON \ -D MPI_BASE_DIR:PATH="/home/rppawlo/local" \ -D MPIEXEC_MAX_NUMPROCS:STRING=4 \ diff --git a/packages/panzer/maintenance/build_panzer_s924793.sh b/packages/panzer/maintenance/build_panzer_s924793.sh index f675726b2fd9..d25696e5cd3e 100755 --- a/packages/panzer/maintenance/build_panzer_s924793.sh +++ b/packages/panzer/maintenance/build_panzer_s924793.sh @@ -7,7 +7,6 @@ cmake \ -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ -D Trilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON \ -D Trilinos_ENABLE_EXAMPLES:BOOL=OFF \ - -D Trios_ENABLE_XDMF:BOOL=OFF \ -D Trilinos_ENABLE_TESTS:BOOL=OFF \ -D Trilinos_ENABLE_Teko:BOOL=ON \ -D Trilinos_ENABLE_Panzer:BOOL=ON \ diff --git a/packages/trios/.gitignore b/packages/trios/.gitignore deleted file mode 100644 index 0b67450a4bd2..000000000000 --- a/packages/trios/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.project -.cproject -.settings diff --git a/packages/trios/CMakeLists.txt b/packages/trios/CMakeLists.txt deleted file mode 100644 index da6a91355ea5..000000000000 --- a/packages/trios/CMakeLists.txt +++ /dev/null @@ -1,188 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -########################################################### -# A) Define the package -# - -TRIBITS_PACKAGE_DECL( Trios ENABLE_SHADOWING_WARNINGS ) - -# Trios package is deprecated April 2019; to be removed May 2019 -MESSAGE(WARNING "***** WARNING: Package Trios is deprecated in April 2019; it will be removed in May 2019 *****") - -TRIBITS_ADD_DEBUG_OPTION() - -TRIBITS_ADD_SHOW_DEPRECATED_WARNINGS_OPTION() - - -########################################################### -# B) Set up package-specific options -# - -set(CMAKE_INCLUDE_DIRECTORIES_BEFORE ON) - -IF (NOT CMAKE_HAVE_PTHREAD_H) - IF (TPL_ENABLE_InfiniBand) - SET (Trios_ENABLE_InfiniBand OFF) - SET (Triosnnti_ENABLE_InfiniBand OFF) - UNSET (HAVE_TRIOS_INFINIBAND) - UNSET (HAVE_TRIOSNNTI_INFINIBAND) - - MESSAGE(WARNING "The Infiniband transport requires Pthreads, but it was disabled or not found.") - ENDIF () -ENDIF () - -# MESSAGE(STATUS "TPL_ENABLE_Pthread==${TPL_ENABLE_Pthread}") -# MESSAGE(STATUS "TPL_ENABLE_InfiniBand==${TPL_ENABLE_InfiniBand}") -# -# MESSAGE(STATUS "Trios_ENABLE_InfiniBand==${Trios_ENABLE_InfiniBand}") -# MESSAGE(STATUS "Triosnnti_ENABLE_InfiniBand==${Triosnnti_ENABLE_InfiniBand}") -# -# MESSAGE(STATUS "CMAKE_HAVE_PTHREAD_H==${CMAKE_HAVE_PTHREAD_H}") -# MESSAGE(STATUS "${PACKAGE_NAME}_HAVE_PTHREAD==${${PACKAGE_NAME}_HAVE_PTHREAD}") -# -# MESSAGE(STATUS "HAVE_TRIOS_INFINIBAND==${HAVE_TRIOS_INFINIBAND}") -# MESSAGE(STATUS "HAVE_TRIOSNNTI_INFINIBAND==${HAVE_TRIOSNNTI_INFINIBAND}") - -# Various Trios checks -INCLUDE(TriosProbeTimers) -# Checks for a supported interconnect -INCLUDE(TriosProbeNetwork) -# Checks for a POSIX threads implementation -INCLUDE(TriosProbeThreads) -# Checks for a POSIX semaphore implementation -INCLUDE(TriosProbeSemaphores) -# Checks for the gettid() system call -INCLUDE(TriosProbeGettid) -# Check XDR implementation -INCLUDE(TriosProbeXDR) - -IF (TPL_ENABLE_Netcdf) - TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_NETCDF_SERVICE - TRIOS_ENABLE_NETCDF_SERVICE - "Build libraries for netcdf service." - OFF ) -ENDIF (TPL_ENABLE_Netcdf) - -IF (TPL_ENABLE_Pnetcdf) -TRIBITS_ADD_OPTION_AND_DEFINE(${PACKAGE_NAME}_ENABLE_PNETCDF_SERVICE - TRIOS_ENABLE_PNETCDF_SERVICE - "Build libraries for PnetCDF service." - OFF ) -ENDIF (TPL_ENABLE_Pnetcdf) - -TRIBITS_ADD_OPTION_AND_DEFINE( ${PACKAGE_NAME}_ENABLE_TRACING - TRIOS_ENABLE_TRACING - "Enable the tracing API" - OFF -) - -# Check for an RDMA transport (needed for Nessie) -IF (${PACKAGE_NAME}_ENABLE_Portals OR - ${PACKAGE_NAME}_ENABLE_CrayPortals OR - ${PACKAGE_NAME}_ENABLE_InfiniBand OR - ${PACKAGE_NAME}_ENABLE_Gemini OR - ${PACKAGE_NAME}_ENABLE_BGPDCMF OR - ${PACKAGE_NAME}_ENABLE_BGQPAMI OR - ${PACKAGE_NAME}_ENABLE_MPI) - - SET(${PACKAGE_NAME}_HAVE_RDMA_TRANSPORT 1) -ELSE() - SET(${PACKAGE_NAME}_HAVE_RDMA_TRANSPORT 0) -ENDIF() - -IF (${PROJECT_NAME}_ENABLE_Trioscommsplitter) - SET(TRIOS_ENABLE_COMMSPLITTER TRUE) -ENDIF() - -# Create the Trios_config.h file in ${CMAKE_BINARY_DIR} -TRIBITS_CONFIGURE_FILE(${PACKAGE_NAME}_config.h) - -# Copy the TriosProcessXDR.cmake file -CONFIGURE_FILE( - ${PACKAGE_SOURCE_DIR}/cmake/TriosProcessXDR.cmake - ${CMAKE_CURRENT_BINARY_DIR}/TriosProcessXDR.cmake - COPYONLY -) - -# Copy the UseTrios.cmake file -CONFIGURE_FILE( - ${PACKAGE_SOURCE_DIR}/cmake/UseTrios.cmake - ${CMAKE_CURRENT_BINARY_DIR}/UseTrios.cmake - COPYONLY -) - - -########################################################### -# C) Add the libraries, tests, and examples -# - -# Need this include so codes can find the config file -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - -TRIBITS_PROCESS_SUBPACKAGES() - -TRIBITS_PACKAGE_DEF() - - -########################################################### -# D) Do standard postprocessing -# - -# Install cmake scripts so others can use them -INSTALL(FILES - ${CMAKE_CURRENT_BINARY_DIR}/TriosProcessXDR.cmake - ${CMAKE_CURRENT_BINARY_DIR}/UseTrios.cmake - DESTINATION "${${PROJECT_NAME}_INSTALL_LIB_DIR}/cmake/${PACKAGE_NAME}" -) - -MESSAGE(STATUS "Trios_TPL_LIBRARIES = ${TPL_LIBRARIES}") - - -TRIBITS_PACKAGE_POSTPROCESS() - -#get_cmake_property(_variableNames VARIABLES) -#foreach (_variableName ${_variableNames}) -# message(STATUS "${_variableName}=${${_variableName}}") -#endforeach() - diff --git a/packages/trios/Copyright.txt b/packages/trios/Copyright.txt deleted file mode 100644 index 326016ba7224..000000000000 --- a/packages/trios/Copyright.txt +++ /dev/null @@ -1,40 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER diff --git a/packages/trios/ReleaseNotes.txt b/packages/trios/ReleaseNotes.txt deleted file mode 100644 index a9723462b7f5..000000000000 --- a/packages/trios/ReleaseNotes.txt +++ /dev/null @@ -1,13 +0,0 @@ ------------------------------------------ -Release Notes for Trilinos Package Trios ------------------------------------------ - -Trilinos 10.12 ------------------------------- - -* Significant changes to thread support for nessie. - * Implemented thread-safe version of nessie client/server... not fully tested. - * Removed thread pool and threaded support from server. Assume service developer has their own thread pool. -* Valgrind-detected bug fixes for IB port of NNTI. -* Modified xfer-service (example) to use multiple servers. - * Implemented two different client/partitioning schemes: round robin, block partition. diff --git a/packages/trios/cmake/Dependencies.cmake b/packages/trios/cmake/Dependencies.cmake deleted file mode 100644 index 4be08aff4a8a..000000000000 --- a/packages/trios/cmake/Dependencies.cmake +++ /dev/null @@ -1,19 +0,0 @@ -SET(SUBPACKAGES_DIRS_CLASSIFICATIONS_OPTREQS - commsplitter libraries/commsplitter EX OPTIONAL - support libraries/support EX OPTIONAL - nnti libraries/nessie/nnti EX OPTIONAL - nssi libraries/nessie/nssi EX OPTIONAL - programs programs EX OPTIONAL - examples examples EX OPTIONAL - tests tests EX OPTIONAL - netcdf-service services/netcdf EX OPTIONAL -) - -SET(LIB_REQUIRED_DEP_PACKAGES ) -SET(LIB_OPTIONAL_DEP_PACKAGES TeuchosCore) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS BGQPAMI BGPDCMF MPI Netcdf Pnetcdf Pthread CrayPortals Portals Gemini InfiniBand Pablo PAPI HPCToolkit) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/cmake/TriosProbeGettid.cmake b/packages/trios/cmake/TriosProbeGettid.cmake deleted file mode 100644 index 0576daa12118..000000000000 --- a/packages/trios/cmake/TriosProbeGettid.cmake +++ /dev/null @@ -1,33 +0,0 @@ - -########## CHECK FOR HEADER FILES ############ - -INCLUDE(CheckIncludeFiles) - -# Probe for syscall header files -CHECK_INCLUDE_FILES("unistd.h" HAVE_TRIOS_UNISTD_H) -CHECK_INCLUDE_FILES("syscall.h" HAVE_TRIOS_SYSCALL_H) - -########## CHECK FOR FUNCTIONS ############ - -INCLUDE(CheckSymbolExists) -INCLUDE(CheckFunctionExists) -INCLUDE(CheckCSourceCompiles) - -# syscall() -IF (HAVE_TRIOS_UNISTD_H) - CHECK_FUNCTION_EXISTS(syscall HAVE_TRIOS_SYSCALL) -ENDIF (HAVE_TRIOS_UNISTD_H) - -# SYS_gettid -IF (HAVE_TRIOS_SYSCALL_H) - CHECK_SYMBOL_EXISTS(SYS_gettid "syscall.h" HAVE_TRIOS_SYS_GETTID) -ENDIF (HAVE_TRIOS_SYSCALL_H) - -IF (HAVE_TRIOS_SYSCALL) - IF (HAVE_TRIOS_SYS_GETTID) - check_c_source_compiles( - "#include \n#include \nint main(){syscall(SYS_gettid);return 0;}" - HAVE_TRIOS_GETTID - ) - ENDIF (HAVE_TRIOS_SYS_GETTID) -ENDIF (HAVE_TRIOS_SYSCALL) diff --git a/packages/trios/cmake/TriosProbeNetwork.cmake b/packages/trios/cmake/TriosProbeNetwork.cmake deleted file mode 100644 index decc7ee4f036..000000000000 --- a/packages/trios/cmake/TriosProbeNetwork.cmake +++ /dev/null @@ -1,101 +0,0 @@ - -include(CheckLibraryExists) -include(CheckFunctionExists) -include(CheckCSourceCompiles) -include(CheckCXXSourceCompiles) -include(CheckTypeSize) - - -########## INCLUDE FILES ############## -include(CheckIncludeFiles) - -check_include_files("malloc.h" HAVE_TRIOS_MALLOC_H) -check_include_files("sys/types.h" HAVE_TRIOS_SYS_TYPES_H) -check_include_files("sys/param.h" HAVE_TRIOS_SYS_PARAM_H) -check_include_files("sys/ioctl.h" HAVE_TRIOS_SYS_IOCTL_H) -check_include_files("sys/socket.h" HAVE_TRIOS_SYS_SOCKET_H) -check_include_files("sys/sockio.h" HAVE_TRIOS_SYS_SOCKIO_H) -check_include_files("netdb.h" HAVE_TRIOS_NETDB_H) -check_include_files("sys/socket;net/if.h" HAVE_TRIOS_NET_IF_H) -check_include_files("sys/socket.h;net/if_dl.h" HAVE_TRIOS_NET_IF_DL_H) -check_include_files("sys/socket.h;net/if_arp.h" HAVE_TRIOS_NET_IF_ARP_H) -check_include_files("netinet/in.h" HAVE_TRIOS_NETINET_IN_H) -check_include_files("arpa/inet.h" HAVE_TRIOS_ARPA_INET_H) -check_include_files("sys/types.h;sys/socket.h;ifaddrs.h" HAVE_TRIOS_IFADDRS_H) - -########## Probe for various network configurations ############## - - -check_function_exists(getifaddrs TRIOS_HAVE_GETIFADDRS) - -set(CMAKE_EXTRA_INCLUDE_FILES "netinet/in.h") -check_type_size("struct sockaddr_in" HAVE_TRIOS_STRUCT_SOCKADDR_IN) -set(CMAKE_EXTRA_INCLUDE_FILES) - - -######## Portals Config ################## - -IF (${PACKAGE_NAME}_ENABLE_Portals OR ${PACKAGE_NAME}_ENABLE_CrayPortals) - - message(STATUS "Checking Portals configuration: PORTALS_IFACE_DEFAULT=${PORTALS_IFACE_DEFAULT}") - - # Need this macro defined to get the right integer types for Portals - ADD_DEFINITIONS(-D__STDC_CONSTANT_MACROS) - - # Also need values for PTL_IFACE_SERVER and PTL_IFACE_CLIENT - if (NOT PORTALS_IFACE_SERVER) - set(PORTALS_IFACE_SERVER "PTL_IFACE_DEFAULT" CACHE STRING "Default NAL for the Portals server") - endif (NOT PORTALS_IFACE_SERVER) - if (NOT PORTALS_IFACE_CLIENT) - set(PORTALS_IFACE_CLIENT "PTL_IFACE_DEFAULT" CACHE STRING "Default NAL for a Portals client") - endif (NOT PORTALS_IFACE_CLIENT) - - ADD_DEFINITIONS(-DPTL_IFACE_CLIENT=${PORTALS_IFACE_CLIENT}) - ADD_DEFINITIONS(-DPTL_IFACE_SERVER=${PORTALS_IFACE_SERVER}) - - # Figure out the name of the Portals include file - IF (${PACKAGE_NAME}_ENABLE_Portals) - SET (PORTALS_INCLUDE_FILE "portals3.h") - ENDIF (${PACKAGE_NAME}_ENABLE_Portals) - - IF (${PACKAGE_NAME}_ENABLE_CrayPortals) - SET (PORTALS_INCLUDE_FILE "portals/portals3.h") - ENDIF (${PACKAGE_NAME}_ENABLE_CrayPortals) - - SET(CMAKE_REQUIRED_INCLUDES ${Portals_INCLUDE_DIRS}) - SET(CMAKE_EXTRA_INCLUDE_FILES ${PORTALS_INCLUDE_FILE}) - set(CMAKE_REQUIRED_LIBRARIES ${TPL_Portals_LIBRARIES}) - - check_include_files(p3nal_utcp.h HAVE_TRIOS_P3NAL_UTCP_H) - check_include_files("p3nal_utcp.h" HAVE_TRIOS_P3NAL_UTCP_H) - check_include_files("p3rt/p3rt.h" HAVE_TRIOS_P3RT_P3RT_H) - - # Check for PTL_NOACK_REQ - check_c_source_compiles( - "#include<${PORTALS_INCLUDE_FILE}>\nmain(){return PTL_NOACK_REQ;}" - HAVE_TRIOS_PTL_NOACK_REQ) - - # Check for PTL_NO_ACK_REQ - check_c_source_compiles( - "#include<${PORTALS_INCLUDE_FILE}>\nint main() {return PTL_NO_ACK_REQ;}" - HAVE_TRIOS_PTL_NO_ACK_REQ) - - # Check for portals types - - check_type_size(ptl_time_t HAVE_TRIOS_PTL_TIME_T) - check_type_size(ptl_eq_handler_t HAVE_TRIOS_PTL_EQ_HANDLER_T) - - # Check for portals functions - check_function_exists(PtlErrorStr HAVE_TRIOS_PTLERRORSTR) - check_function_exists(PtlNIFailStr HAVE_TRIOS_PTLNIFAILSTR) - check_function_exists(PtlEventKindStr HAVE_TRIOS_PTLEVENTKINDSTR) - check_function_exists(PtlGetJid HAVE_TRIOS_PTLGITJID) - check_function_exists(PtlACEntry HAVE_TRIOS_PTLACENTRY) - - SET(CMAKE_REQUIRED_INCLUDES) - SET(CMAKE_EXTRA_INCLUDE_FILES) - set(CMAKE_REQUIRED_LIBRARIES) - -ENDIF (${PACKAGE_NAME}_ENABLE_Portals OR ${PACKAGE_NAME}_ENABLE_CrayPortals) - - diff --git a/packages/trios/cmake/TriosProbeSemaphores.cmake b/packages/trios/cmake/TriosProbeSemaphores.cmake deleted file mode 100644 index 2c473e9c0dfe..000000000000 --- a/packages/trios/cmake/TriosProbeSemaphores.cmake +++ /dev/null @@ -1,81 +0,0 @@ -########## CHECK FOR HEADER FILES ############ - -INCLUDE(CheckIncludeFiles) - -# Probe for semaphore header file -CHECK_INCLUDE_FILES("semaphore.h" HAVE_TRIOS_SEMAPHORE_H) - -########## CHECK FOR FUNCTIONS ############ - -include(CheckCSourceCompiles) -INCLUDE(CheckCSourceRuns) - - -IF (HAVE_TRIOS_SEMAPHORE_H) - - check_c_source_compiles( - "#include \nint main(){sem_t lock;return 0;}" - HAVE_TRIOS_SEM_T - ) - - # Probe for unnamed semaphore implementation - SET(SOURCE - " - #include - #include - int main() - { - sem_t lock; - if (sem_init(&lock, 0, 1) == -1) { - return(1); - } - if (sem_wait(&lock) == -1) { - return(2); - } - if (sem_post(&lock) == -1) { - return(3); - } - if (sem_destroy(&lock) == -1) { - return(4); - } - - return 0; - } - " - ) - - CHECK_C_SOURCE_RUNS("${SOURCE}" HAVE_TRIOS_UNNAMED_SEMAPHORES) - - # Probe for named semaphore implementation - SET(SOURCE - " - #include - #include - int main() - { - sem_t *lock; - lock=sem_open(\"/trios.sem_test\", O_CREAT, 0600, 1); - if (lock == SEM_FAILED) { - return(1); - } - if (sem_wait(lock) == -1) { - return(2); - } - if (sem_post(lock) == -1) { - return(3); - } - if (sem_close(lock) == -1) { - return(4); - } - if (sem_unlink(\"/trios.sem_test\") == -1) { - return(5); - } - - return 0; - } - " - ) - - CHECK_C_SOURCE_RUNS("${SOURCE}" HAVE_TRIOS_NAMED_SEMAPHORES) - -ENDIF(HAVE_TRIOS_SEMAPHORE_H) diff --git a/packages/trios/cmake/TriosProbeThreads.cmake b/packages/trios/cmake/TriosProbeThreads.cmake deleted file mode 100644 index 51a95216f505..000000000000 --- a/packages/trios/cmake/TriosProbeThreads.cmake +++ /dev/null @@ -1,83 +0,0 @@ -include(CheckLibraryExists) -include(CheckFunctionExists) -include(CheckCSourceCompiles) -include(CheckCXXSourceCompiles) - -########## Probe for various thread configurations ############## - -IF (CMAKE_HAVE_PTHREAD_H) - - # Probe for pthreads header file - CHECK_INCLUDE_FILES("pthread.h" HAVE_TRIOS_PTHREAD_H) - - - # Test for a way to yield besides usleep(0) - check_c_source_compiles( - "#include \nint main(){pthread_yield();return 0;}" - HAVE_TRIOS_PTHREAD_YIELD - ) - check_c_source_compiles( - "#include \nint main(){pthread_yield_np();return 0;}" - HAVE_TRIOS_PTHREAD_YIELD_NP - ) - check_c_source_compiles( - "#include \nint main(){sched_yield();return 0;}" - HAVE_TRIOS_SCHED_YIELD - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;return 0;}" - HAVE_TRIOS_PTHREAD_MUTEX_T - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;pthread_mutex_init(&mutex,NULL);return 0;}" - HAVE_TRIOS_PTHREAD_MUTEX_INIT - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;pthread_mutex_lock(&mutex);return 0;}" - HAVE_TRIOS_PTHREAD_MUTEX_LOCK - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;pthread_mutex_unlock(&mutex);return 0;}" - HAVE_TRIOS_PTHREAD_MUTEX_UNLOCK - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;pthread_mutex_destroy(&mutex);return 0;}" - HAVE_TRIOS_PTHREAD_MUTEX_DESTROY - ) - check_c_source_compiles( - "#include \nint main(){pthread_cond_t cond;return 0;}" - HAVE_TRIOS_PTHREAD_COND_T - ) - check_c_source_compiles( - "#include \nint main(){pthread_cond_t cond;pthread_cond_init(&cond,NULL);return 0;}" - HAVE_TRIOS_PTHREAD_COND_INIT - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;pthread_cond_t cond;pthread_cond_wait(&cond,&mutex);return 0;}" - HAVE_TRIOS_PTHREAD_COND_WAIT - ) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex;pthread_cond_t cond;struct timespec *abstime;pthread_cond_timedwait(&cond,&mutex,abstime);return 0;}" - HAVE_TRIOS_PTHREAD_COND_TIMEDWAIT - ) - check_c_source_compiles( - "#include \nint main(){pthread_cond_t cond;pthread_cond_signal(&cond);return 0;}" - HAVE_TRIOS_PTHREAD_COND_SIGNAL - ) - check_c_source_compiles( - "#include \nint main(){pthread_cond_t cond;pthread_cond_broadcast(&cond);return 0;}" - HAVE_TRIOS_PTHREAD_COND_BROADCAST - ) - check_c_source_compiles( - "#include \nint main(){pthread_cond_t cond;pthread_cond_destroy(&cond);return 0;}" - HAVE_TRIOS_PTHREAD_COND_DESTROY - ) - add_definitions(-D_GNU_SOURCE) # needed to find RECURSIVE mutex initializer - # Test for PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP - set(CMAKE_REQUIRED_DEFINITIONS -D_GNU_SOURCE) - check_c_source_compiles( - "#include \nint main(){pthread_mutex_t mutex=PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;return 0;}" - HAVE_TRIOS_PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP - ) - -ENDIF() diff --git a/packages/trios/cmake/TriosProbeTimers.cmake b/packages/trios/cmake/TriosProbeTimers.cmake deleted file mode 100644 index af715e5ee664..000000000000 --- a/packages/trios/cmake/TriosProbeTimers.cmake +++ /dev/null @@ -1,44 +0,0 @@ - -########## CHECK FOR HEADER FILES ############ - -INCLUDE(CheckIncludeFiles) - -# Probe for timer header files -CHECK_INCLUDE_FILES("mach/mach_time.h" HAVE_TRIOS_MACH_TIME_H) -CHECK_INCLUDE_FILES("time.h" HAVE_TRIOS_TIME_H) -CHECK_INCLUDE_FILES("sys/time.h" HAVE_TRIOS_SYS_TIME_H) - -if (TPL_ENABLE_PAPI) - CHECK_INCLUDE_FILES("papi.h" HAVE_TRIOS_PAPI_H) -endif (TPL_ENABLE_PAPI) - -########## CHECK FOR FUNCTIONS ############ - -INCLUDE(CheckLibraryExists) -INCLUDE(CheckFunctionExists) -INCLUDE(CheckCSourceCompiles) -INCLUDE(CheckCXXSourceCompiles) - -# clock_gettime -IF (HAVE_TRIOS_TIME_H) - CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_TRIOS_CLOCK_GETTIME) - IF (HAVE_TRIOS_CLOCK_GETTIME) - SET(CMAKE_REQUIRED_LIBRARIES "rt;${CMAKE_REQUIRED_LIBRARIES}") - ENDIF (HAVE_TRIOS_CLOCK_GETTIME) -ENDIF(HAVE_TRIOS_TIME_H) - -# gettimeofday -IF (HAVE_TRIOS_SYS_TIME_H) - CHECK_FUNCTION_EXISTS(gettimeofday HAVE_TRIOS_GETTIMEOFDAY) -ENDIF (HAVE_TRIOS_SYS_TIME_H) - -# papi -IF (HAVE_TRIOS_PAPI_H) - CHECK_FUNCTION_EXISTS(PAPI_get_real_usec HAVE_TRIOS_PAPI_GET_REAL_USEC) - SET(HAVE_TRIOS_PAPI ${HAVE_TRIOS_PAPI_GET_REAL_USEC}) -ENDIF (HAVE_TRIOS_PAPI_H) - -# mach_absolute_time (on apple) -IF (HAVE_TRIOS_MACH_TIME_H) - CHECK_FUNCTION_EXISTS(mach_absolute_time HAVE_TRIOS_MACH_ABSOLUTE_TIME) -ENDIF (HAVE_TRIOS_MACH_TIME_H) diff --git a/packages/trios/cmake/TriosProbeXDR.cmake b/packages/trios/cmake/TriosProbeXDR.cmake deleted file mode 100644 index 19d799a6f95f..000000000000 --- a/packages/trios/cmake/TriosProbeXDR.cmake +++ /dev/null @@ -1,21 +0,0 @@ - -########## CHECK FOR HEADER FILES ############ - -INCLUDE(CheckIncludeFiles) - -########## CHECK FOR FUNCTIONS ############ - -INCLUDE(CheckLibraryExists) -INCLUDE(CheckFunctionExists) -INCLUDE(CheckCSourceCompiles) -INCLUDE(CheckCXXSourceCompiles) - -CHECK_FUNCTION_EXISTS(xdr_u_int16_t HAVE_TRIOS_XDR_U_INT16_T) -CHECK_FUNCTION_EXISTS(xdr_u_int32_t HAVE_TRIOS_XDR_U_INT32_T) -CHECK_FUNCTION_EXISTS(xdr_u_int64_t HAVE_TRIOS_XDR_U_INT64_T) - -# XDR_SIZEOF -check_c_source_compiles( - "#include \nint main(){xdr_sizeof(NULL,NULL);return 0;}" - HAVE_TRIOS_XDR_SIZEOF -) diff --git a/packages/trios/cmake/TriosProcessXDR.cmake b/packages/trios/cmake/TriosProcessXDR.cmake deleted file mode 100644 index c1f39b44059d..000000000000 --- a/packages/trios/cmake/TriosProcessXDR.cmake +++ /dev/null @@ -1,26 +0,0 @@ -############# Function to generate XDR source and header Files ############### - -function (TriosProcessXDR path) - - GET_FILENAME_COMPONENT(file ${path} NAME_WE) - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${file}.c - COMMAND rpcgen -Cc ${path} - | sed -e "\"s#include.*${file}.*#include <${file}.h>#\"" - > ${CMAKE_CURRENT_BINARY_DIR}/${file}.c - DEPENDS ${path} ${file}.h) - - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${file}.h - COMMAND rpcgen -Ch ${path} - | sed -e "\"s#rpc/rpc.h#${file}.h#\"" - | perl -pe \"BEGIN{undef $$/\;} s/\(enum\\s\\w+\\s\\{\\n\(\\s*.*?,\\n\)*?\\s*.*?\),\(\\n\\s*\\}\;\)/\\1\\3/smg\" - > ${CMAKE_CURRENT_BINARY_DIR}/${file}.h - DEPENDS ${path}) - - # Need target to force construction of nssi_types_xdr.{c,h} and nnti_xdr.{c,h} - add_custom_target(${file} ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${file}.c ${CMAKE_CURRENT_BINARY_DIR}/${file}.h) - -endfunction (TriosProcessXDR) diff --git a/packages/trios/cmake/Trios_config.h.in b/packages/trios/cmake/Trios_config.h.in deleted file mode 100644 index cab1bfac849f..000000000000 --- a/packages/trios/cmake/Trios_config.h.in +++ /dev/null @@ -1,134 +0,0 @@ - -/* Deprecation notice */ -#if !defined(TRILINOS_HIDE_DEPRECATED_HEADER_WARNINGS) -#ifdef __GNUC__ -# warning "The package Trios is deprecated in April 2019; it will be removed from Trilinos in May 2019." -#endif -#endif - -/* ************************************************ - * Network configuration - * ************************************************ */ - -/* Network Transports */ -#cmakedefine HAVE_TRIOS_CRAYPORTALS 1 -#cmakedefine HAVE_TRIOS_PORTALS 1 -#cmakedefine HAVE_TRIOS_INFINIBAND 1 -#cmakedefine HAVE_TRIOS_GEMINI 1 -#cmakedefine HAVE_TRIOS_BGPDCMF 1 -#cmakedefine HAVE_TRIOS_BGQPAMI 1 -#cmakedefine HAVE_TRIOS_MPI 1 - -/* Special Portals config */ -#cmakedefine HAVE_TRIOS_PTLERRORSTR 1 -#cmakedefine HAVE_TRIOS_PTLNIFAILSTR 1 -#cmakedefine HAVE_TRIOS_PTLEVENTKINDSTR 1 -#cmakedefine HAVE_TRIOS_PTLGITJID 1 -#cmakedefine HAVE_TRIOS_PTLACENTRY 1 -#cmakedefine HAVE_TRIOS_PTL_NOACK_REQ 1 -#cmakedefine HAVE_TRIOS_PTL_NO_ACK_REQ 1 -#cmakedefine HAVE_TRIOS_P3RT_P3RT_H 1 -#cmakedefine HAVE_TRIOS_P3NAL_UTCP_H 1 -#cmakedefine HAVE_TRIOS_PTL_TIME_T -#cmakedefine HAVE_TRIOS_PTL_EQ_HANDLER_T - -/* Headers */ -#cmakedefine HAVE_TRIOS_MALLOC_H 1 -#cmakedefine HAVE_TRIOS_SYS_TYPES_H 1 -#cmakedefine HAVE_TRIOS_SYS_PARAM_H 1 -#cmakedefine HAVE_TRIOS_SYS_IOCTL_H 1 -#cmakedefine HAVE_TRIOS_SYS_SOCKET_H 1 -#cmakedefine HAVE_TRIOS_SYS_SOCKIO_H 1 -#cmakedefine HAVE_TRIOS_NETDB_H 1 -#cmakedefine HAVE_TRIOS_NET_IF_H 1 -#cmakedefine HAVE_TRIOS_NET_IF_DL_H 1 -#cmakedefine HAVE_TRIOS_NET_IF_ARP_H 1 -#cmakedefine HAVE_TRIOS_NETINET_IN_H 1 -#cmakedefine HAVE_TRIOS_ARPA_INET_H 1 -#cmakedefine HAVE_TRIOS_IFADDRS_H 1 -#cmakedefine HAVE_TRIOS_STRUCT_SOCKADDR_IN 1 - -/* ************************************************ - * I/O Libraries - * ************************************************ */ -#cmakedefine HAVE_TRIOS_NETCDF 1 -#cmakedefine HAVE_TRIOS_PNETCDF 1 - -/* ************************************************ - * Misc Config - * ************************************************ */ -#cmakedefine TRIOS_HAVE_GETIFADDRS 1 - -/* ************************************************ - * CommSplitter Config - * ************************************************ */ -#cmakedefine TRIOS_ENABLE_COMMSPLITTER 1 - - -/* ************************************************ - * Semaphore configuration - * ************************************************ */ -#cmakedefine HAVE_TRIOS_SEMAPHORE_H 1 -#cmakedefine HAVE_TRIOS_SEM_T 1 -#cmakedefine HAVE_TRIOS_UNNAMED_SEMAPHORES 1 -#cmakedefine HAVE_TRIOS_NAMED_SEMAPHORES 1 - - -/* ************************************************ - * Linux thread ID configuration - * ************************************************ */ -#cmakedefine HAVE_TRIOS_UNISTD_H 1 -#cmakedefine HAVE_TRIOS_SYSCALL_H 1 -#cmakedefine HAVE_TRIOS_GETTID 1 - - -/* ************************************************ - * Pthread configuration - * ************************************************ */ -#cmakedefine HAVE_TRIOS_PTHREAD_H 1 -#cmakedefine HAVE_TRIOS_PTHREAD_MUTEX_T 1 -#cmakedefine HAVE_TRIOS_PTHREAD_MUTEX_INIT 1 -#cmakedefine HAVE_TRIOS_PTHREAD_MUTEX_LOCK 1 -#cmakedefine HAVE_TRIOS_PTHREAD_MUTEX_UNLOCK 1 -#cmakedefine HAVE_TRIOS_PTHREAD_MUTEX_DESTROY 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_T 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_INIT 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_WAIT 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_TIMEDWAIT 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_SIGNAL 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_BROADCAST 1 -#cmakedefine HAVE_TRIOS_PTHREAD_COND_DESTROY 1 - - -/* ************************************************ - * Timer configuration - * ************************************************ */ -#cmakedefine HAVE_TRIOS_SYS_TIME_H 1 -#cmakedefine HAVE_TRIOS_CLOCK_GETTIME 1 -#cmakedefine HAVE_TRIOS_TIME_H 1 -#cmakedefine HAVE_TRIOS_GETTIMEOFDAY 1 -#cmakedefine HAVE_TRIOS_PAPI_H 1 -#cmakedefine HAVE_TRIOS_PAPI 1 -#cmakedefine HAVE_TRIOS_MACH_TIME_H 1 -#cmakedefine HAVE_TRIOS_MACH_ABSOLUTE_TIME 1 - -/* ************************************************ - * Tracing configuration - * ************************************************ */ -#cmakedefine TRIOS_ENABLE_TRACING 1 -#cmakedefine TRIOS_ENABLE_PAPI @Trios_ENABLE_PAPI@ -#cmakedefine HAVE_TRIOS_PABLO 1 - -/* ************************************************ - * HPCToolkit configuration - * ************************************************ */ -#cmakedefine HAVE_TRIOS_HPCTOOLKIT 1 - -/* ************************************************ - * XDR configuration - * ************************************************ */ -#cmakedefine HAVE_TRIOS_XDR_U_INT16_T -#cmakedefine HAVE_TRIOS_XDR_U_INT32_T -#cmakedefine HAVE_TRIOS_XDR_U_INT64_T -#cmakedefine HAVE_TRIOS_XDR_SIZEOF 1 - diff --git a/packages/trios/cmake/UseTrios.cmake b/packages/trios/cmake/UseTrios.cmake deleted file mode 100644 index b13db62c4508..000000000000 --- a/packages/trios/cmake/UseTrios.cmake +++ /dev/null @@ -1,52 +0,0 @@ -# -# -# See the file TriosConfig.cmake for all variables set when -# executing the FindPackage(Trios) command. -# -# Compilers -# Trios_CXX_COMPILER -# Trios_C_COMPILER -# Trios_FORTRAN_COMPILER -# -# Compiler Flags -# Trios_CXX_FLAGS -# Trios_C_FLAGS -# Trios_FORTRAN_FLAGS -# Trios_EXTRA_LD_FLAGS -# -# Paths -# Trios_INCLUDE_DIRS -# Trios_LIBRARY_DIRS -# Trios_LIBRARIES -# Trios_TPL_INCLUDE_DIRS -# Trios_TPL_LIBRARY_DIRS -# Trios_TPL_LIBRARIES -# -IF(NOT Trios_USE_FILE_INCLUDED) - SET(Trios_USE_FILE_INCLUDED 1) - - if (Trios_FOUND) - - # Add compiler flags - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${Trios_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${Trios_CXX_FLAGS}") - SET(CMAKE_FORTRAN_FLAGS "${CMAKE_FORTRAN_FLAGS} ${Trios_FORTRAN_FLAGS}") - - # Add include directories - INCLUDE_DIRECTORIES(${Trios_INCLUDE_DIRS}) - INCLUDE_DIRECTORIES(${Trios_TPL_INCLUDE_DIRS}) - - # Add link directories - LINK_DIRECTORIES(${Trios_LIBRARY_DIRS}) - LINK_DIRECTORIES(${Trios_TPL_LIBRARY_DIRS}) - - - # Add libraries directories - SET(CMAKE_EXTRA_LIBS "${CMAKE_EXTRA_LIBS}") - - # Add path to CMAKE files - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Trios_DIR}") - - ENDIF (Trios_FOUND) - -ENDIF(NOT Trios_USE_FILE_INCLUDED) diff --git a/packages/trios/docs/.gitignore b/packages/trios/docs/.gitignore deleted file mode 100644 index 2ec816f3ef2c..000000000000 --- a/packages/trios/docs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -html -latex diff --git a/packages/trios/docs/Doxyfile b/packages/trios/docs/Doxyfile deleted file mode 100644 index 22db094dd2dc..000000000000 --- a/packages/trios/docs/Doxyfile +++ /dev/null @@ -1,1750 +0,0 @@ -# Doxyfile 1.7.5.1 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the config file -# that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# http://www.gnu.org/software/libiconv for the list of possible encodings. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or sequence of words) that should -# identify the project. Note that if you do not use Doxywizard you need -# to put quotes around the project name if it contains spaces. - -PROJECT_NAME = "Network Scalable Service Interface (Nessie)" - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer -# a quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = - -# With the PROJECT_LOGO tag one can specify an logo or icon that is -# included in the documentation. The maximum height of the logo should not -# exceed 55 pixels and the maximum width should not exceed 200 pixels. -# Doxygen will copy the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = - -# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create -# 4096 sub-directories (in 2 levels) under the output directory of each output -# format and will distribute the generated files over these directories. -# Enabling this option can be useful when feeding doxygen a huge amount of -# source files, where putting all generated files in the same directory would -# otherwise cause performance problems for the file system. - -CREATE_SUBDIRS = NO - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, -# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, -# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English -# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, -# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, -# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. - -OUTPUT_LANGUAGE = English - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator -# that is used to form the text in various listings. Each string -# in this list, if found as the leading text of the brief description, will be -# stripped from the text and the result after processing the whole list, is -# used as the annotated text. Otherwise, the brief description is used as-is. -# If left blank, the following values are used ("$name" is automatically -# replaced with the name of the entity): "The $name class" "The $name widget" -# "The $name file" "is" "provides" "specifies" "contains" -# "represents" "a" "an" "the" - -ABBREVIATE_BRIEF = - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the -# path to strip. - -STRIP_FROM_PATH = - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of -# the path mentioned in the documentation of a class, which tells -# the reader which header file to include in order to use a class. -# If left blank only the name of the header file containing the class -# definition is used. Otherwise one should specify the include paths that -# are normally passed to the compiler using the -I flag. - -STRIP_FROM_INC_PATH = - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful if your file system -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like regular Qt-style comments -# (thus requiring an explicit @brief command for a brief description.) - -JAVADOC_AUTOBRIEF = NO - -# If the QT_AUTOBRIEF tag is set to YES then Doxygen will -# interpret the first line (until the first dot) of a Qt-style -# comment as the brief description. If set to NO, the comments -# will behave just like regular Qt-style comments (thus requiring -# an explicit \brief command for a brief description.) - -QT_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# re-implements. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce -# a new page for each member. If set to NO, the documentation of a member will -# be part of the file/class/namespace that contains it. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C -# sources only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = YES - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java -# sources only. Doxygen will then generate output that is more tailored for -# Java. For instance, namespaces will be presented as packages, qualified -# scopes will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources only. Doxygen will then generate output that is more tailored for -# Fortran. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for -# VHDL. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given extension. -# Doxygen has a built-in mapping, but you can override or extend it using this -# tag. The format is ext=language, where ext is a file extension, and language -# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, -# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make -# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C -# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions -# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should -# set this tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. -# func(std::string) {}). This also makes the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. - -BUILTIN_STL_SUPPORT = NO - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. -# Doxygen will parse them like normal C++ but will assume all classes use public -# instead of private inheritance when no explicit protection keyword is present. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate getter -# and setter methods for a property. Setting this option to YES (the default) -# will make doxygen replace the get and set methods by a property in the -# documentation. This will only work if the methods are indeed getting or -# setting a simple type. If this is not the case, or you want to show the -# methods anyway, you should set this option to NO. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and -# unions are shown inside the group in which they are included (e.g. using -# @ingroup) instead of on a separate page (for HTML and Man pages) or -# section (for LaTeX and RTF). - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and -# unions with only public data fields will be shown inline in the documentation -# of the scope in which they are defined (i.e. file, namespace, or group -# documentation), provided this scope is documented. If set to NO (the default), -# structs, classes, and unions are shown on a separate page (for HTML and Man -# pages) or section (for LaTeX and RTF). - -INLINE_SIMPLE_STRUCTS = NO - -# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum -# is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically -# be useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. - -TYPEDEF_HIDES_STRUCT = NO - -# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to -# determine which symbols to keep in memory and which to flush to disk. -# When the cache is full, less often used symbols will be written to disk. -# For small to medium size projects (<1000 input files) the default value is -# probably good enough. For larger projects a too small cache size can cause -# doxygen to be busy swapping symbols to and from disk most of the time -# causing a significant performance penalty. -# If the system has enough physical memory increasing the cache will improve the -# performance by keeping more symbols in memory. Note that the value works on -# a logarithmic scale so increasing the size by one will roughly double the -# memory usage. The cache size is given by this formula: -# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, -# corresponding to a cache size of 2^16 = 65536 symbols - -SYMBOL_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. When set to YES local -# methods, which are defined in the implementation section but not in -# the interface are included in the documentation. -# If set to NO (the default) only methods in the interface are included. - -EXTRACT_LOCAL_METHODS = NO - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base -# name of the file that contains the anonymous namespace. By default -# anonymous namespaces are hidden. - -EXTRACT_ANON_NSPACES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen -# will list include files with double quotes in the documentation -# rather than with sharp brackets. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = NO - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the -# brief documentation of file, namespace and class members alphabetically -# by member name. If set to NO (the default) the members will appear in -# declaration order. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen -# will sort the (brief and detailed) documentation of class members so that -# constructors and destructors are listed first. If set to NO (the default) -# the constructors will appear in the respective orders defined by -# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. -# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO -# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the -# hierarchy of group names into alphabetical order. If set to NO (the default) -# the group names will appear in their defined order. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be -# sorted by fully-qualified names, including namespaces. If set to -# NO (the default), the class list will be sorted only by class name, -# not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the -# alphabetical list. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to -# do proper type resolution of all parameters of a function it will reject a -# match between the prototype and the implementation of a member function even -# if there is only one candidate or it is obvious which candidate to choose -# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen -# will still accept a match between prototype and implementation in such cases. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = NO - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = NO - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = NO - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= NO - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or macro consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and macros in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# If the sources in your project are distributed over multiple directories -# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy -# in the documentation. The default is NO. - -SHOW_DIRECTORIES = NO - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. -# This will remove the Files entry from the Quick Index and from the -# Folder Tree View (if specified). The default is YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the -# Namespaces page. This will remove the Namespaces entry from the Quick Index -# and from the Folder Tree View (if specified). The default is YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command , where is the value of -# the FILE_VERSION_FILTER tag, and is the name of an input file -# provided by doxygen. Whatever the program writes to standard output -# is used as the file version. See the manual for examples. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. The create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. -# You can optionally specify a file name after the option, if omitted -# DoxygenLayout.xml will be used as the name of the layout file. - -LAYOUT_FILE = DoxygenLayout.xml - -# The CITE_BIB_FILES tag can be used to specify one or more bib files -# containing the references data. This must be a list of .bib files. The -# .bib extension is automatically appended if omitted. Using this command -# requires the bibtex tool to be installed. See also -# http://en.wikipedia.org/wiki/BibTeX for more info. For LaTeX the style -# of the bibliography can be controlled using LATEX_BIB_STYLE. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_NO_PARAMDOC option can be enabled to get warnings for -# functions that are documented, but have no documentation for their parameters -# or return value. If set to NO (the default) doxygen will only warn about -# wrong or incomplete parameter documentation, but not about the absence of -# documentation. - -WARN_NO_PARAMDOC = NO - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. Optionally the format may contain -# $version, which will be replaced by the version of the file (if it could -# be obtained via FILE_VERSION_FILTER) - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = ../ - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is -# also the default input encoding. Doxygen uses libiconv (or the iconv built -# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for -# the list of possible encodings. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh -# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py -# *.f90 *.f *.for *.vhd *.vhdl - -FILE_PATTERNS = *.h \ - *.doxy \ - *.x - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# Note that relative paths are relative to directory from which doxygen is run. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. Note that the wildcards are matched -# against the file with absolute path, so to exclude all test directories -# for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test - -EXCLUDE_SYMBOLS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = images - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command , where -# is the value of the INPUT_FILTER tag, and is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. If FILTER_PATTERNS is specified, this tag will be -# ignored. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: -# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further -# info on how filters are used. If FILTER_PATTERNS is empty or if -# non of the patterns match the file name, INPUT_FILTER is applied. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) -# and it is also possible to disable source filtering for a specific pattern -# using *.ext= (so without naming a filter). This option only has effect when -# FILTER_SOURCE_FILES is enabled. - -FILTER_SOURCE_PATTERNS = - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. -# Note: To get rid of all source code in the generated output, make sure also -# VERBATIM_HEADERS is set to NO. - -SOURCE_BROWSER = YES - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = NO - -# If the REFERENCES_RELATION tag is set to YES -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = NO - -# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) -# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from -# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will -# link to the source code. Otherwise they will link to the documentation. - -REFERENCES_LINK_SOURCE = YES - -# If the USE_HTAGS tag is set to YES then the references to source code -# will point to the HTML generated by the htags(1) tool instead of doxygen -# built-in source browser. The htags tool is part of GNU's global source -# tagging system (see http://www.gnu.org/software/global/global.html). You -# will need version 4.8.6 or higher. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = YES - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. Note that when using a custom header you are responsible -# for the proper inclusion of any scripts and style sheets that doxygen -# needs, which is dependent on the configuration options used. -# It is adviced to generate a default header using "doxygen -w html -# header.html footer.html stylesheet.css YourConfigFile" and then modify -# that header. Note that the header is subject to change so you typically -# have to redo this when upgrading to a newer version of doxygen or when -# changing the value of configuration settings such as GENERATE_TREEVIEW! - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet. Note that doxygen will try to copy -# the style sheet file to the HTML output directory, so don't put your own -# stylesheet in the HTML output directory as well, or it will be erased! - -HTML_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that -# the files will be copied as-is; there are no commands or markers available. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. -# Doxygen will adjust the colors in the stylesheet and background images -# according to this color. Hue is specified as an angle on a colorwheel, -# see http://en.wikipedia.org/wiki/Hue for more information. -# For instance the value 0 represents red, 60 is yellow, 120 is green, -# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. -# The allowed range is 0 to 359. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of -# the colors in the HTML output. For a value of 0 the output will use -# grayscales only. A value of 255 will produce the most vivid colors. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to -# the luminance component of the colors in the HTML output. Values below -# 100 gradually make the output lighter, whereas values above 100 make -# the output darker. The value divided by 100 is the actual gamma applied, -# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, -# and 100 does not change the gamma. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting -# this to NO can help when comparing the output of multiple runs. - -HTML_TIMESTAMP = YES - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. For this to work a browser that supports -# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox -# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). - -HTML_DYNAMIC_SECTIONS = NO - -# If the GENERATE_DOCSET tag is set to YES, additional index files -# will be generated that can be used as input for Apple's Xcode 3 -# integrated development environment, introduced with OSX 10.5 (Leopard). -# To create a documentation set, doxygen will generate a Makefile in the -# HTML output directory. Running make will produce the docset in that -# directory and running "make install" will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find -# it at startup. -# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html -# for more information. - -GENERATE_DOCSET = NO - -# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the -# feed. A documentation feed provides an umbrella under which multiple -# documentation sets from a single provider (such as a company or product suite) -# can be grouped. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that -# should uniquely identify the documentation set bundle. This should be a -# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen -# will append .docset to the name. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output directory. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING -# is used to encode HtmlHelp index (hhk), content (hhc) and project file -# content. - -CHM_INDEX_ENCODING = - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated -# that can be used as input for Qt's qhelpgenerator to generate a -# Qt Compressed Help (.qch) of the generated HTML documentation. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can -# be used to specify the file name of the resulting .qch file. -# The path specified is relative to the HTML output folder. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating -# Qt Help Project output. For more information please see -# http://doc.trolltech.com/qthelpproject.html#namespace - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating -# Qt Help Project output. For more information please see -# http://doc.trolltech.com/qthelpproject.html#virtual-folders - -QHP_VIRTUAL_FOLDER = doc - -# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to -# add. For more information please see -# http://doc.trolltech.com/qthelpproject.html#custom-filters - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see -# -# Qt Help Project / Custom Filters. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's -# filter section matches. -# -# Qt Help Project / Filter Attributes. - -QHP_SECT_FILTER_ATTRS = - -# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can -# be used to specify the location of Qt's qhelpgenerator. -# If non-empty doxygen will try to run qhelpgenerator on the generated -# .qhp file. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files -# will be generated, which together with the HTML files, form an Eclipse help -# plugin. To install this plugin and make it available under the help contents -# menu in Eclipse, the contents of the directory containing the HTML and XML -# files needs to be copied into the plugins directory of eclipse. The name of -# the directory within the plugins directory should be the same as -# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before -# the help appears. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have -# this name. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values -# (range [0,1..20]) that doxygen will group on one line in the generated HTML -# documentation. Note that a value of 0 will completely suppress the enum -# values from appearing in the overview section. - -ENUM_VALUES_PER_LINE = 4 - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. -# If the tag value is set to YES, a side panel will be generated -# containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, -# and Class Hierarchy pages using a tree view instead of an ordered list. - -USE_INLINE_TREES = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open -# links to external symbols imported via tag files in a separate window. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of Latex formulas included -# as images in the HTML documentation. The default is 10. Note that -# when you change the font size after a successful doxygen run you need -# to manually remove any form_*.png images from the HTML output directory -# to force them to be regenerated. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are -# not supported properly for IE 6.0, but are supported on all modern browsers. -# Note that when changing this option you need to delete any form_*.png files -# in the HTML output before the changes have effect. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax -# (see http://www.mathjax.org) which uses client side Javascript for the -# rendering instead of using prerendered bitmaps. Use this if you do not -# have LaTeX installed or if you want to formulas look prettier in the HTML -# output. When enabled you also need to install MathJax separately and -# configure the path to it using the MATHJAX_RELPATH option. - -USE_MATHJAX = NO - -# When MathJax is enabled you need to specify the location relative to the -# HTML output directory using the MATHJAX_RELPATH option. The destination -# directory should contain the MathJax.js script. For instance, if the mathjax -# directory is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the -# mathjax.org site, so you can quickly see the result without installing -# MathJax, but it is strongly recommended to install a local copy of MathJax -# before deployment. - -MATHJAX_RELPATH = http://www.mathjax.org/mathjax - -# The MATHJAX_EXTENSIONS tag can be used to specify one or MathJax extension -# names that should be enabled during MathJax rendering. - -MATHJAX_EXTENSIONS = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box -# for the HTML output. The underlying search engine uses javascript -# and DHTML and should work on any modern browser. Note that when using -# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets -# (GENERATE_DOCSET) there is already a search function so this one should -# typically be disabled. For large projects the javascript based search engine -# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. - -SEARCHENGINE = YES - -# When the SERVER_BASED_SEARCH tag is enabled the search engine will be -# implemented using a PHP enabled web server instead of at the web client -# using Javascript. Doxygen will generate the search PHP script and index -# file to put on the web server. The advantage of the server -# based approach is that it scales better to large projects and allows -# full text search. The disadvantages are that it is more difficult to setup -# and does not have live searching capabilities. - -SERVER_BASED_SEARCH = NO - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. -# Note that when enabling USE_PDFLATEX this option is only used for -# generating bitmaps for formulas in the HTML output, but not in the -# Makefile that is written to the output directory. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = YES - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = letter - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for -# the generated latex document. The footer should contain everything after -# the last chapter. If it is left blank doxygen will generate a -# standard footer. Notice: only use this tag if you know what you are doing! - -LATEX_FOOTER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = YES - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = YES - -# If LATEX_SOURCE_CODE is set to YES then doxygen will include -# source code with syntax highlighting in the LaTeX output. -# Note that which sources are shown also depends on other settings -# such as SOURCE_BROWSER. - -LATEX_SOURCE_CODE = YES - -# The LATEX_BIB_STYLE tag can be used to specify the style to use for the -# bibliography, e.g. plainnat, or ieeetr. The default style is "plain". See -# http://en.wikipedia.org/wiki/BibTeX for more info. - -LATEX_BIB_STYLE = plain - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimized for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assignments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -# If the XML_PROGRAMLISTING tag is set to YES Doxygen will -# dump the program listings (including syntax highlighting -# and cross-referencing information) to the XML output. Note that -# enabling this will significantly increase the size of the XML output. - -XML_PROGRAMLISTING = YES - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = NO - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_DEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# pointed to by INCLUDE_PATH will be searched when a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. To prevent a macro definition from being -# undefined via #undef or recursively expanded use the := operator -# instead of the = operator. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition that -# overrules the definition found in the source code. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all references to function-like macros -# that are alone on a line, have an all uppercase name, and do not end with a -# semicolon, because these will confuse the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base -# or super classes. Setting the tag to NO turns the diagrams off. Note that -# this option also works with HAVE_DOT disabled, but it is recommended to -# install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = NO - -# You can define message sequence charts within doxygen comments using the \msc -# command. Doxygen will then run the mscgen tool (see -# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the -# documentation. The MSCGEN_PATH tag allows you to specify the directory where -# the mscgen tool resides. If left empty the tool is assumed to be found in the -# default search path. - -MSCGEN_PATH = - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is -# allowed to run in parallel. When set to 0 (the default) doxygen will -# base this on the number of processors available in the system. You can set it -# explicitly to a value larger than 0 to get control over the balance -# between CPU load and processing speed. - -DOT_NUM_THREADS = 0 - -# By default doxygen will use the Helvetica font for all dot files that -# doxygen generates. When you want a differently looking font you can specify -# the font name using DOT_FONTNAME. You need to make sure dot is able to find -# the font, which can be done by putting it in a standard location or by setting -# the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the -# directory containing the font. - -DOT_FONTNAME = FreeSans.ttf - -# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. -# The default size is 10pt. - -DOT_FONTSIZE = 10 - -# By default doxygen will tell dot to use the Helvetica font. -# If you specify a different font using DOT_FONTNAME you can use DOT_FONTPATH to -# set the path where dot can find it. - -DOT_FONTPATH = - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = NO - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for groups, showing the direct groups dependencies - -GROUP_GRAPHS = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similar to the OMG's Unified Modeling -# Language. - -UML_LOOK = NO - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = NO - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT options are set to YES then -# doxygen will generate a call dependency graph for every global function -# or class method. Note that enabling this option will significantly increase -# the time of a run. So in most cases it will be better to enable call graphs -# for selected functions only using the \callgraph command. - -CALL_GRAPH = NO - -# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then -# doxygen will generate a caller dependency graph for every global function -# or class method. Note that enabling this option will significantly increase -# the time of a run. So in most cases it will be better to enable caller -# graphs for selected functions only using the \callergraph command. - -CALLER_GRAPH = NO - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will generate a graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES -# then doxygen will show the dependencies a directory has on other directories -# in a graphical way. The dependency relations are determined by the #include -# relations between the files in the directories. - -DIRECTORY_GRAPH = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are svg, png, jpg, or gif. -# If left blank png will be used. If you choose svg you need to set -# HTML_FILE_EXTENSION to xhtml in order to make the SVG files -# visible in IE 9+ (other browsers do not have this requirement). - -DOT_IMAGE_FORMAT = png - -# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to -# enable generation of interactive SVG images that allow zooming and panning. -# Note that this requires a modern browser other than Internet Explorer. -# Tested and working are Firefox, Chrome, Safari, and Opera. For IE 9+ you -# need to set HTML_FILE_EXTENSION to xhtml in order to make the SVG files -# visible. Older versions of IE do not have SVG support. - -INTERACTIVE_SVG = NO - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found in the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the -# \mscfile command). - -MSCFILE_DIRS = - -# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of -# nodes that will be shown in the graph. If the number of nodes in a graph -# becomes larger than this value, doxygen will truncate the graph, which is -# visualized by representing a node as a red box. Note that doxygen if the -# number of direct children of the root node in a graph is already larger than -# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note -# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. - -DOT_GRAPH_MAX_NODES = 50 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes -# that lay further from the root node will be omitted. Note that setting this -# option to 1 or 2 may greatly reduce the computation time needed for large -# code bases. Also note that the size of a graph can be further restricted by -# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. - -MAX_DOT_GRAPH_DEPTH = 0 - -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not -# seem to support this out of the box. Warning: Depending on the platform used, -# enabling this option may lead to badly anti-aliased labels on the edges of -# a graph (i.e. they become hard to read). - -DOT_TRANSPARENT = NO - -# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output -# files in one run (i.e. multiple -o and -T options on the command line). This -# makes dot run faster, but since only newer versions of dot (>1.8.10) -# support this, this feature is disabled by default. - -DOT_MULTI_TARGETS = NO - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES diff --git a/packages/trios/examples/CMakeLists.txt b/packages/trios/examples/CMakeLists.txt deleted file mode 100644 index 9338fe371d82..000000000000 --- a/packages/trios/examples/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ - - -TRIBITS_SUBPACKAGE(examples) - -TRIBITS_ADD_EXAMPLE_DIRECTORIES(xfer-service) - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/examples/cmake/Dependencies.cmake b/packages/trios/examples/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/examples/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/examples/xfer-service/CMakeLists.txt b/packages/trios/examples/xfer-service/CMakeLists.txt deleted file mode 100644 index 5da387f34658..000000000000 --- a/packages/trios/examples/xfer-service/CMakeLists.txt +++ /dev/null @@ -1,106 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - - - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - -# Generate the XDR header and source file for the service arguments -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/xfer_service_args.x) - - -TRIBITS_ADD_EXECUTABLE( - XferMPITest - SOURCES xfer_service_args.c xfer_util.cpp xfer_mpi_server.cpp xfer_mpi_test.cpp xfer_mpi_client.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - - -TRIBITS_ADD_EXECUTABLE( - XferServiceTest - SOURCES xfer_service_args.c xfer_util.cpp xfer_server.cpp xfer_threads.cpp xfer_service_test.cpp xfer_client.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -SET(EXTRA_TEST_ARGS "") - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -SET(TESTNAMES - write-encode-sync - write-encode-async - write-rdma-sync - write-rdma-async - read-encode-sync - read-encode-async - read-rdma-sync - read-rdma-async) - - -foreach (testname ${TESTNAMES}) - - SET(DEFAULT_ARGS "--io-method=${testname} --validate ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file=${testname}-result.out ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file-mode=a ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-trials=5 ") - - TRIBITS_ADD_TEST( - XferServiceTest - NOEXEPREFIX - NAME XferService_${testname} - ARGS "${DEFAULT_ARGS} ${EXTRA_TEST_ARGS}" - COMM serial mpi - NUM_MPI_PROCS 2) - - #SET_TESTS_PROPERTIES(${PACKAGE_NAME}_XferService_${testname}_MPI_2 PROPERTIES TIMEOUT 10) -endforeach() - - diff --git a/packages/trios/examples/xfer-service/cmake/Dependencies.cmake b/packages/trios/examples/xfer-service/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/examples/xfer-service/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/examples/xfer-service/xfer_client.cpp b/packages/trios/examples/xfer-service/xfer_client.cpp deleted file mode 100644 index 9e11d8476755..000000000000 --- a/packages/trios/examples/xfer-service/xfer_client.cpp +++ /dev/null @@ -1,932 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file xfer-client.cpp - * - */ - -/** - * @defgroup xfer_client Data Transfer Client - * - * @ingroup xfer_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "xfer_client.h" - - -#include -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - -#include "xfer_service_args.h" -#include "xfer_util.h" - - - -log_level client_debug_level = LOG_UNDEFINED; - -/* prototype for a function to initialize buffers */ -extern int print_args( - std::ostream &out, - const struct xfer_args &args, - const char *prefix); - - - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_write_encode( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array, - nssi_request *req) -{ - int rc = NSSI_OK; - xfer_write_encode_args args; - - /* initialize the arguments */ - memset(&args, 0, sizeof(xfer_write_encode_args)); - args.len = len; - args.seed = seed; - args.validate = validate; - - args.array.data_array_t_len = array->data_array_t_len; - args.array.data_array_t_val = array->data_array_t_val; - - /* call the remote methods */ - rc = nssi_call_rpc(svc, XFER_WRITE_ENCODE_OP, &args, NULL, 0, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call xfer_write_encode: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of data with the request structure. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int xfer_write_encode_blk( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = xfer_write_encode(svc, len, seed, validate, array, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - -/** - * @brief Asynchronous transfer of data via RDMA. - * - * This method sends the length of the data array to the server. After - * receiving the request, the server pulls the data from the memory - * buffer using the nssi_get_data() function. This function is asynchronous -- - * after sending the request to the server, this function returns the - * request data structure. The caller must at some point call the - * nssi_wait() or nssi_test() function to find out if the request is - * complete. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The service request data structure. - */ -int xfer_write_rdma( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array, - nssi_request *req) -{ - int rc = NSSI_OK; - xfer_write_rdma_args args; - int nbytes; - - /* the buffer to send to the server */ - const data_t *buf = array->data_array_t_val; - - /* initialize the arguments */ - memset(&args, 0, sizeof(xfer_write_rdma_args)); - - /* the only argument to send is the size of the array */ - args.len = len; - args.seed = seed; - args.validate = validate; - - /* calculate the number of bytes in the buffer */ - nbytes = args.len*sizeof(data_t); - - /* call the remote methods (send buffer using the data portal) */ - rc = nssi_call_rpc(svc, XFER_WRITE_RDMA_OP, &args, (char *)buf, nbytes, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call xfer_write_encode: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of data via RDMA. - * - * This method sends the length of the data array to the server. After - * receiving the request, the server pulls the data from the memory - * buffer using the nssi_get_data() function. This function is blocking -- - * it calls the nssi_wait() function to make sure the server completed - * the request before returning. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int xfer_write_rdma_blk( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = xfer_write_rdma(svc, len, seed, validate, array, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - - - -/** - * @brief Asynchronous transfer of \ref data_t array via RDMA. The server sends the data back in the result. - * - * This method sends the length of the data array to the server. After - * receiving the request, the server pulls the data from the memory - * buffer using the nssi_get_data() function. This function is asynchronous -- - * after sending the request to the server, this function returns the - * request data structure. The caller must at some point call the - * nssi_wait() or nssi_test() function to find out if the request is - * complete. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_read_encode( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - xfer_read_encode_res *res, - nssi_request *req) -{ - int rc = NSSI_OK; - xfer_read_encode_args args; - - /* initialize the arguments */ - memset(&args, 0, sizeof(xfer_read_encode_args)); - memset(res, 0, sizeof(xfer_read_encode_res)); - - /* initialize the arguments */ - args.len = len; - args.seed = seed; - args.validate = validate; - - /* call the remote methods, array comes back in res */ - rc = nssi_call_rpc(svc, XFER_READ_ENCODE_OP, &args, NULL, 0, res, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call xfer_4: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array via RDMA. The server sends the data back in the result. - * - * This method sends the length of the data array to the server. After - * receiving the request, the server pulls the data from the memory - * buffer using the nssi_get_data() function. This function is ssynchronous -- - * after sending the request to the server, this function calls nssi_wait() - * to find out the result of the request. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int xfer_read_encode_blk( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - xfer_read_encode_res *res) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = xfer_read_encode(svc, len, seed, validate, res, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. The server sends the data back via RDMA. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. The server is expected to send the data - * back using nssi_put_data(). - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_read_rdma( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - data_array_t *put_buf, - nssi_request *req) -{ - int rc = NSSI_OK; - xfer_read_rdma_args args; - log_level debug_level = client_debug_level; - - int nbytes; - - /* initialize the arguments */ - memset(&args, 0, sizeof(xfer_read_rdma_args)); - - assert(len == put_buf->data_array_t_len); - - - /* tell the server how about the buffer */ - args.len = len; - args.seed = seed; - args.validate = validate; - - /* calculate the number of bytes in the buffer */ - nbytes = args.len*sizeof(data_t); - - data_t *buf; - //buf = (data_t *)malloc(nbytes); - buf = put_buf->data_array_t_val; - - log_debug(debug_level, "Calling RPC for XFER_READ_RDMA_OP, len=%d, val=%p, nbytes=%d", - put_buf->data_array_t_len, buf, nbytes); - - /* call the remote methods */ - rc = nssi_call_rpc(svc, XFER_READ_RDMA_OP, &args, buf, nbytes, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call xfer_5: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array with the request structure. The server sends the data back in the result. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int xfer_read_rdma_blk( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - data_array_t *put_buf) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = xfer_read_rdma(svc, len, seed, validate, put_buf, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - - - -int read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return(-1); - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return(1); - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(client_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); - - return(0); -} - - - - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param xfer_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -xfer_client_main (struct xfer_args &args, nssi_service &xfer_svc, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int i,j; - double time; - double start_time; - std::ofstream result_stream; - log_level debug_level = args.debug_level; - int client_rank, client_size; - - /* the array of results (for async experiments) */ - std::vector results; - - xfer_read_encode_res *res4=NULL; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - std::vector < nssi_request > reqs(num_reqs); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(XFER_WRITE_ENCODE_OP, xfer_write_encode_args, void, void); - NSSI_REGISTER_CLIENT_STUB(XFER_WRITE_RDMA_OP, xfer_write_rdma_args, void, void); - NSSI_REGISTER_CLIENT_STUB(XFER_READ_ENCODE_OP, xfer_read_encode_args, void, xfer_read_encode_res); - NSSI_REGISTER_CLIENT_STUB(XFER_READ_RDMA_OP, xfer_read_rdma_args, void, void); - - - /* allocate space for the data arrays used for testing */ - std::vector array_vec; - array_vec.reserve(args.num_reqs); // async jobs need num_reqs arrays - - for (i=0; iarray, &array_vec[j]); - if (rc != 0) { - log_error(client_debug_level, "Validation failed"); - MPI_Abort(MPI_COMM_WORLD, rc); - } - } - - if (res4->array.data_array_t_val) free(res4->array.data_array_t_val); - } - - break; - } - - case XFER_READ_ENCODE_ASYNC: - { - /* submit requests */ - res4 = (xfer_read_encode_res *)malloc(args.num_reqs * sizeof(xfer_read_encode_res)); - for (j=0; j -#include - -#include "Trios_logger.h" - - - -enum IO_METHODS { - XFER_WRITE_ENCODE_SYNC = 0, - XFER_WRITE_ENCODE_ASYNC, - XFER_WRITE_RDMA_SYNC, - XFER_WRITE_RDMA_ASYNC, - XFER_READ_ENCODE_SYNC, - XFER_READ_ENCODE_ASYNC, - XFER_READ_RDMA_SYNC, - XFER_READ_RDMA_ASYNC -}; - -enum DIST_SCHEME { - XFER_BLOCK_DISTRIBUTION, - XFER_ROUND_ROBIN_DISTRIBUTION -}; - -enum MPI_IO_METHODS { - XFER_MPI_SEND=0, - XFER_MPI_ISEND, - XFER_MPI_RECV, - XFER_MPI_IRECV, - XFER_MPI_PUT, - XFER_MPI_GET -}; - - -/** - * Options and arguments passed to the client driver. - */ -struct xfer_args { - bool client_flag; - bool server_flag; - int num_servers; // used for the server only - int num_threads; // used for the server only - bool block_distribution; // how to assign clients to servers - std::string server_url; - int transport; - std::string transport_name; - int len; - int io_method; - std::string url_file; - std::string io_method_name; - log_level debug_level; - std::string logfile; - int num_trials; - int num_reqs; - std::string result_file; - std::string result_file_mode; - int timeout; - int delay; - int num_retries; - bool validate_flag; - bool kill_server_flag; -}; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - - -#else /* K&R C */ -#endif - - - - -#ifdef __cplusplus -} -#endif - - -#endif /* XFER_CLIENT_H_ */ diff --git a/packages/trios/examples/xfer-service/xfer_debug.cpp b/packages/trios/examples/xfer-service/xfer_debug.cpp deleted file mode 100644 index 0cee02c12dc8..000000000000 --- a/packages/trios/examples/xfer-service/xfer_debug.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file xfer_debug.cpp - * - */ - -#include "Trios_logger.h" -#include "xfer_debug.h" - -/** - * Initialize the xfer_debug_level variable to LOG_UNDEFINED. This - * tells the logger to use the value set by logger_init, if called. - */ -log_level xfer_debug_level = LOG_UNDEFINED; diff --git a/packages/trios/examples/xfer-service/xfer_debug.h b/packages/trios/examples/xfer-service/xfer_debug.h deleted file mode 100644 index 8e794e0405b6..000000000000 --- a/packages/trios/examples/xfer-service/xfer_debug.h +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * xfer_client.h - * - * Created on: Aug 22, 2011 - * Author: raoldfi - */ - -#ifndef XFER_DEBUG_H_ -#define XFER_DEBUG_H_ - -#include "Trios_logger.h" - -extern log_level xfer_debug_level; - -#endif /* XFER_CLIENT_H_ */ diff --git a/packages/trios/examples/xfer-service/xfer_mpi_client.cpp b/packages/trios/examples/xfer-service/xfer_mpi_client.cpp deleted file mode 100644 index 3c35e2be3235..000000000000 --- a/packages/trios/examples/xfer-service/xfer_mpi_client.cpp +++ /dev/null @@ -1,567 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file xfer-client.cpp - * - */ - -/** - * @defgroup xfer_client Data Transfer Client - * - * @ingroup xfer_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "xfer_client.h" -#include "xfer_debug.h" -#include "xfer_util.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - -#include "xfer_service_args.h" - - - -log_level client_debug_level = LOG_UNDEFINED; - - -extern int print_args( - std::ostream &out, - const struct xfer_args &args, - const char *prefix); - - - -/** - * @brief Transfer an array of data structures using the MPI send/receive functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_mpi_send( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - data_array_t *array) -{ - int rc = 0; - MPI_Status status; - log_level debug_level = xfer_debug_level; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - int req_buf[3]; - req_buf[0] = len; - req_buf[1] = seed; - req_buf[2] = (int)validate; - - int nbytes = len * sizeof(struct data_t); - - log_debug(debug_level, "%d: Sending send req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - - // The request is just an MPI send with the number of elements - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, XFER_MPI_SEND_REQ_TAG, MPI_COMM_WORLD); - - log_debug(debug_level, "%d: Waiting for ack from %d to before sending data", rank, server_rank); - // Receive an ack that the server is ready - MPI_Recv(&rc, 0, MPI_INT, server_rank, XFER_MPI_SEND_ACK_TAG, MPI_COMM_WORLD, &status); - - - log_debug(debug_level, "%d: Send data: %d structs, %d bytes to %d", rank, len, nbytes, server_rank); - - // Now we send the data - rc = MPI_Send(&array->data_array_t_val[0], nbytes, - MPI_BYTE, server_rank, XFER_MPI_SEND_DATA_TAG, MPI_COMM_WORLD); - - return rc; -} - - -/** - * @brief Transfer an array of data structures using the MPI send/receive functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_mpi_isend( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array, - MPI_Request *req) -{ - int rc = 0; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Status status; - log_level debug_level = xfer_debug_level; - - int req_buf[3]; - req_buf[0] = (int)len; - req_buf[1] = (int)seed; - req_buf[2] = (int)validate; - - int nbytes = len * sizeof(data_t); - - log_debug(debug_level, "%d: Sending isend req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, XFER_MPI_ISEND_REQ_TAG, MPI_COMM_WORLD); - - // Receive an ack that the server is ready - MPI_Recv(&rc, 0, MPI_INT, server_rank, XFER_MPI_ISEND_ACK_TAG, MPI_COMM_WORLD, &status); - - log_debug(debug_level, "%d: Isend data: %d structs, %d bytes", rank, len, nbytes); - rc = MPI_Isend(array->data_array_t_val, nbytes, MPI_BYTE, server_rank, XFER_MPI_ISEND_DATA_TAG, MPI_COMM_WORLD, req); - - return rc; -} - - -/** - * @brief Transfer an array of data structures using the MPI put functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_mpi_put( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = 0; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Win win; - - int req_buf[3]; - req_buf[0] = (int)len; - req_buf[1] = (int)seed; - req_buf[2] = (int)validate; - int nbytes = len * sizeof(data_t); - - log_level debug_level = xfer_debug_level; - - - log_debug(debug_level, "%d: Sending put req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, XFER_MPI_PUT_REQ_TAG, MPI_COMM_WORLD); - - // The window on this side is not used to receive data - MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); - - MPI_Win_fence(0, win); - - - log_debug(debug_level, "%d: Put data: %d structs, %d bytes, val[0]=%d", rank, len, nbytes, - array->data_array_t_val[0].int_val); - - // Put the data from this window into a buffer on the server - MPI_Put(array->data_array_t_val, nbytes, MPI_BYTE, server_rank, 0, nbytes, MPI_BYTE, win); - - // This makes sure the operations are complete - MPI_Win_fence(0, win); - - MPI_Win_free(&win); - - return rc; -} - - -/** - * @brief Transfer an array of data structures using the MPI put functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int xfer_mpi_get( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = 0; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Win win; - - int req_buf[3]; - req_buf[0] = (int)len; - req_buf[1] = (int)seed; - req_buf[2] = (int)validate; - int nbytes = len * sizeof(data_t); - - log_level debug_level = xfer_debug_level; - - - log_debug(debug_level, "%d: Sending put req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, XFER_MPI_GET_REQ_TAG, MPI_COMM_WORLD); - - // The window on this side is not used to receive data - MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); - - MPI_Win_fence(0, win); - - - log_debug(debug_level, "%d: Get data: %d structs, %d bytes, val[0]=%d", rank, len, nbytes, - array->data_array_t_val[0].int_val); - - // Put the data from this window into a buffer on the server - MPI_Get(array->data_array_t_val, nbytes, MPI_BYTE, server_rank, 0, nbytes, MPI_BYTE, win); - - // This makes sure the operations are complete - MPI_Win_fence(0, win); - - MPI_Win_free(&win); - - return rc; -} - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param xfer_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -xfer_mpi_client_main (struct xfer_args &args, const int server_rank, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int i,j; - double time; - double start_time; - std::ofstream result_stream; - log_level debug_level = xfer_debug_level; - int client_rank, client_size; - int global_rank, global_size; - - /* the array of results (for async experiments) */ - std::vector results; - - xfer_read_encode_res *res4=NULL; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); - MPI_Comm_size(MPI_COMM_WORLD, &global_size); - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - log_debug(debug_level, "%d: Starting client", global_rank); - - /* initialize logger */ -// if (args.logfile.empty()) { -// logger_init(args.debug_level, NULL); -// debug_level = args.debug_level; -// } else { -// char fn[1024]; -// sprintf(fn, "%s.%03d.log", args.logfile.c_str(), client_rank); -// logger_init(args.debug_level, fn); -// debug_level = args.debug_level; -// } - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - std::vector < MPI_Request > reqs(num_reqs); - std::vector < MPI_Status > status(num_reqs); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - - /* allocate space for the data arrays used for testing */ - std::vector array_vec; - array_vec.reserve(args.num_reqs); // async jobs need num_reqs arrays - - for (i=0; i -#include - -#include -#include -#include -#include -#include -#include - - - -#include - -log_level xfer_debug_level = LOG_UNDEFINED; - -/** - * @brief Send the data through the MPI_Send function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_send( - const int len, - const int seed, - const int validate, - const int source) -{ - int nbytes = len*sizeof(data_t); - MPI_Request req; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_level debug_level = xfer_debug_level; - int rc = 0; - MPI_Status status; - - log_debug(debug_level, "%d: starting process_mpi_send(len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - - log_debug(debug_level, "%d: Server Posting IRECV for %d structs, %d bytes, tag=%d", - rank, len, nbytes, XFER_MPI_SEND_DATA_TAG); - - // Post a receive for the buffer of ''len'' elements - MPI_Irecv(array.data_array_t_val, nbytes, MPI_BYTE, source, XFER_MPI_SEND_DATA_TAG, MPI_COMM_WORLD, &req); - - log_debug(debug_level, "%d: Server Sending \"READY ACK\" to %d", rank, source); - - // Send an ack to the client that the server is ready - rc = MPI_Send(&rc, 0, MPI_INT, source, XFER_MPI_SEND_ACK_TAG, MPI_COMM_WORLD); - - log_debug(debug_level, "%d: Server is waiting for data from %d, rc=%d", rank, source, rc); - - // Wait for the data to arrive - rc = MPI_Wait(&req, &status); - log_debug(debug_level, "%d: MPI_Wait complete: rc=%d, status.MPI_ERROR=%d, status.MPI_SOURCE=%d", - rank, rc, status.MPI_ERROR, status.MPI_SOURCE); - - if (validate) { - rc = xfer_validate_array(seed, &array); - if (rc != 0) - log_warn(debug_level, "Invalid array"); - } - - // free the memory - delete [] array.data_array_t_val; - - return rc; -} - - -/** - * @brief Send the data through the MPI_Send function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_isend( - const int len, - const int seed, - const int validate, - const int source) -{ - int nbytes = len*sizeof(data_t); - - log_level debug_level = xfer_debug_level; - int rc=0; - MPI_Status status; - MPI_Request req; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(debug_level, "%d: starting process_mpi_isend(len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - log_debug(debug_level, "Posting IRECV"); - - // Post a receive for the buffer of ''len'' elements - MPI_Irecv(array.data_array_t_val, nbytes, MPI_BYTE, source, XFER_MPI_ISEND_DATA_TAG, MPI_COMM_WORLD, &req); - - log_debug(debug_level, "Sending \"READY ACK\""); - - // Send an ack to the client that the server is ready - MPI_Send(&rc, 0, MPI_INT, source, XFER_MPI_ISEND_ACK_TAG, MPI_COMM_WORLD); - - log_debug(debug_level, "Waiting for data"); - - // Wait for the data to arrive - MPI_Wait(&req, &status); - - if (validate) { - rc = xfer_validate_array(seed, &array); - if (rc != 0) - log_warn(debug_level, "Invalid array"); - } - - // Do we need to send a result? - delete [] array.data_array_t_val; - - return rc; -} - - -/** - * @brief Send the data through the MPI_Put function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_put( - const int len, - const int seed, - const int validate, - const int source) -{ - log_level debug_level = xfer_debug_level; - int rc = 0; - MPI_Win win; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(debug_level, "%d: starting process_mpi_put(len=%d, seed=%d, validate=%d)", - rank, len, seed, validate); - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - log_debug(debug_level, "%d: creating window", rank); - - - // Collective call to create a window for the data being transferred (like registering the memory) - MPI_Win_create(array.data_array_t_val, sizeof(data_t)*len, - 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); - - - MPI_Win_fence(0, win); - log_debug(debug_level, "Waiting for data"); - MPI_Win_fence(0, win); - - - log_debug(debug_level, "Data should be here... validate if necessary, val[0]=%d",array.data_array_t_val[0].int_val); - - if (validate) { - rc = xfer_validate_array(seed, &array); - if (rc == 0) { - log_info(debug_level, "Validate Passed"); - } - else { - log_warn(debug_level, "Invalid array"); - } - - log_info(debug_level, "after validate, rc=%d", rc); - } - - - // clean up - MPI_Win_free(&win); - - - delete [] array.data_array_t_val; - - - return rc; -} - - -/** - * @brief Client reads the data using the MPI_Get function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_get( - const int len, - const int seed, - const int validate, - const int source) -{ - log_level debug_level = xfer_debug_level; - int rc = 0; - MPI_Win win; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(debug_level, "%d: starting process_mpi_get(len=%d, seed=%d, validate=%d)", - rank, len, seed, validate); - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - log_debug(debug_level, "%d: creating window", rank); - - // initialize the array - if (validate) { - xfer_init_data_array(seed, &array); - } - - // Collective call to create a window for the data being transferred (like registering the memory) - MPI_Win_create(array.data_array_t_val, sizeof(data_t)*len, - 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); - - - MPI_Win_fence(0, win); - log_debug(debug_level, "Waiting for client to get data"); - MPI_Win_fence(0, win); - - - log_debug(debug_level, "Data should be transferred"); - - - // clean up - MPI_Win_free(&win); - - - delete [] array.data_array_t_val; - - - return rc; -} - - -void make_progress(bool is_idle) -{ - log_debug(xfer_debug_level, "current_time(%llu) is_idle(%llu)", (uint64_t)trios_get_time_ms(), (uint64_t)is_idle); - - return; -} - - -/** - * @brief The NSSI xfer-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int xfer_mpi_server_main(MPI_Comm server_comm) -{ - int rc = 0; - - log_level debug_level = xfer_debug_level; - int server_rank; - - MPI_Comm_rank(server_comm, &server_rank); - - log_debug(debug_level, "%d: Starting server", server_rank); - - /* options that can be overriden by the command-line */ - std::string logfile(""); - - bool done=false; - MPI_Status status; - - // Server loop just waits for requests from any of the clients. The - // server isn't expecting any particular type of request. - - while (!done) { - int req_buf[3]; - - log_debug(debug_level, "Waiting for request"); - - // Receive the next request, the tag identifies the type of request - MPI_Recv(&req_buf, 3, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - int len = req_buf[0]; - int seed = req_buf[1]; - int validate = req_buf[2]; - - log_debug(debug_level, "Server Received Request (source=%d, tag=%d, len=%d, seed=%d, validate=%d)", - status.MPI_SOURCE, status.MPI_TAG, len, seed, validate); - - switch (status.MPI_TAG) { - - case XFER_MPI_SEND_REQ_TAG: - rc = process_mpi_send(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing SEND request"); - break; - - case XFER_MPI_ISEND_REQ_TAG: - log_debug(debug_level, "Server Calling ISEND"); - rc = process_mpi_isend(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing ISEND request"); - break; - - case XFER_MPI_PUT_REQ_TAG: - log_debug(debug_level, "Server Calling PUT"); - rc = process_mpi_put(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing PUT request"); - break; - - case XFER_MPI_GET_REQ_TAG: - log_debug(debug_level, "Server Calling PUT"); - rc = process_mpi_get(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing GET request"); - break; - - case XFER_MPI_FINI_REQ: - log_debug(debug_level, "Server Received EXIT"); - done = true; - break; - - default: - log_error(debug_level, "tag=%d is not supported", status.MPI_TAG); - return -1; - } - } - - log_debug(debug_level, "Exiting server"); - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/examples/xfer-service/xfer_mpi_test.cpp b/packages/trios/examples/xfer-service/xfer_mpi_test.cpp deleted file mode 100644 index 7fc2c96b5d47..000000000000 --- a/packages/trios/examples/xfer-service/xfer_mpi_test.cpp +++ /dev/null @@ -1,372 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - - -/* - * xfer_service_test.cpp - * - * Created on: Aug 22, 2011 - * Author: raoldfi - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include "xfer_client.h" -#include "xfer_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -int xfer_mpi_server_main(MPI_Comm server_comm); -int xfer_mpi_client_main (struct xfer_args &args, const int server_rank, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct xfer_args &args, - const char *prefix) -{ - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url << std::endl; - - if (args.client_flag) { - out << prefix << " \tio-method = " << args.io_method_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tlen = " << args.len << std::endl; - out << prefix << " \tvalidate = " << ((args.validate_flag)?"true":"false") << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct xfer_args args; - - const int num_io_methods = 6; - const int io_method_vals[] = { - XFER_MPI_SEND, XFER_MPI_ISEND, - XFER_MPI_RECV, XFER_MPI_IRECV, - XFER_MPI_PUT, XFER_MPI_GET}; - const char * io_method_names[] = { - "send", "isend", - "recv", "irecv", - "put", "get"}; - - - // Initialize arguments - args.len = 1; - args.delay = 1; - args.io_method = XFER_MPI_SEND; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.timeout = 500; - args.num_retries = 5; - args.validate_flag = true; - args.server_url = ""; - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("len", &args.len, "The number of structures in an input buffer"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("validate", "no-validate", &args.validate_flag, "Validate the data"); - - // Set an enumeration command line option for the io_method - - parser.setOption("io-method", &args.io_method, num_io_methods, io_method_vals, io_method_names, - "I/O Methods for the example: \n" - "\t\t\tsend : Write data using the MPI_Send function\n" - "\t\t\tisend: Write data using the MPI_Isend function \n" - "\t\t\trecv : Read data using MPI_Recv function\n" - "\t\t\tirecv : Read data using the MPI_Irecv function\n" - "\t\t\tput: Write data using the MPI_Put function\n" - "\t\t\tget: Read data through the MPI_Get function"); - - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - debug_level = xfer_debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting xfer-mpi test", rank); - - - /** - * We need to play some fancy MPI games to get the communicators working correctly. - * We're running as both a client and a server in the same app, so we need to - * split the communicator so that the client can barrier sync without involving the server. - */ - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - int server_rank = 0; - int color = (rank == server_rank) ? 0 : 1; // only one server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - -// std::cout << "rank=" << rank << "/" << np << ", color=" << color << -// ", new_rank=" << splitrank << "/" << splitsize << std::endl << std::endl; - - - - // Set the debug level for the xfer service. - xfer_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - args.io_method_name = io_method_names[args.io_method]; - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (rank == server_rank) { - rc = xfer_mpi_server_main(comm); - log_debug(debug_level, "Server is finished, rc=%d", rc); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. */ - else { - log_debug(debug_level, "Starting client main"); - // Start the client code - rc = xfer_mpi_client_main(args, server_rank, comm); - log_debug(debug_level, "Client is finished, rc=%d", rc); - } - - MPI_Barrier(MPI_COMM_WORLD); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - - if (!success) { - log_warn(debug_level, "%d: success=FALSE", rank); - } - if (rc != 0) { - log_warn(debug_level, "%d: rc=%d", rank, rc); - } - - if(success && (rc == 0)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/examples/xfer-service/xfer_server.cpp b/packages/trios/examples/xfer-service/xfer_server.cpp deleted file mode 100644 index 82f36d7f82fb..000000000000 --- a/packages/trios/examples/xfer-service/xfer_server.cpp +++ /dev/null @@ -1,451 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file xfer-server.cpp - * - * @brief Example data transfer server. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - */ - -/** - * @defgroup xfer_server Data Transfer Server - * - * @ingroup xfer_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -//#include "nssi_types.h" -#include "Trios_nssi_server.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nssi_debug.h" - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "xfer_util.h" -#include "xfer_threads.h" - - - -#include - -log_level xfer_debug_level = LOG_UNDEFINED; - - -/** - * @brief Emulate a write operation where all the data is sent - * through the function arguments. - * - * Transfer an array of data structures to the server through the - * procedure arguments, forcing the client to encode the array before - * sending and the server to decode the array when receiving. We use - * this method to evaluate the performance of the encoding/decoding the arguments. - * For large arrays, this method also tests our two-phase transfer protocol in - * which the client pushes a small header of arguments and lets the server pull the - * remaining arguments on demand. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int xfer_write_encode_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const xfer_write_encode_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - const int len = args->len; - const long int seed = args->seed; - const bool validate = args->validate; - - int nbytes = len*sizeof(data_t); - - log_level debug_level = xfer_debug_level; - int rc; - - /* process array (nothing to do) */ - - log_debug(debug_level, "starting xfer_write_encode_srvr"); - - /* Validate the array that was sent through the args */ - if (validate) { - data_array_t tmp_array; - tmp_array.data_array_t_len = len; - - tmp_array.data_array_t_val = (data_t *)malloc(nbytes); - - xfer_init_data_array(seed, &tmp_array); - rc = xfer_compare_data_arrays(&tmp_array, const_cast(&args->array)); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - - free(tmp_array.data_array_t_val); - } - - - - rc = nssi_send_result(caller, request_id, NSSI_OK, NULL, res_addr); - - return rc; -} - -/** - * @brief Transfer an array of data structures to the server using the data channel. - * - * This procedure passes the length of the array in the arguments. The server - * then ``pulls'' the unencoded data from the client using the \ref nssi_get function. - * This method evaluates the RDMA transfer performance for the \ref nssi_get_data function. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data. - * @param res_addr The remote memory descriptor for the result. - */ -int xfer_write_rdma_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const xfer_write_rdma_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NSSI_OK; - log_level debug_level = xfer_debug_level; - - const int len = args->len; - const long int seed = args->seed; - const bool validate = args->validate; - - int nbytes = len*sizeof(data_t); - - log_debug(debug_level, "starting xfer_write_rdma_srvr"); - - /* allocate space for the incoming buffer */ - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = (data_t *)malloc(nbytes); - memset(array.data_array_t_val, 0, nbytes); - - log_debug(debug_level, "getting data from client (%s)", caller->url); - - /* now we need to fetch the data from the client */ - rc = nssi_get_data(caller, array.data_array_t_val, nbytes, data_addr); - if (rc != NSSI_OK) { - log_warn(debug_level, "could not fetch data from client"); - return rc; - } - - /* Validate the array */ - if (validate && (rc == 0)) { - data_array_t tmp_array; - tmp_array.data_array_t_len = len; - tmp_array.data_array_t_val = (data_t *)malloc(nbytes); - - xfer_init_data_array(seed, &tmp_array); - rc = xfer_compare_data_arrays(&tmp_array, &array); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - - free(tmp_array.data_array_t_val); - } - - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - - free(array.data_array_t_val); - - return rc; -} - - - -/** - * @brief Transfer an array of data structures to the client - * using the control channel. - * - * This method tells the server to send the data array to the - * client through the result data structure, forcing the server - * to encode the array before sending and the client to decode the array - * when receiving. This procedure evaluates the performance of the encoding/decoding - * the arguments. For large arrays, this method also tests our two-phase transfer - * protocol for the result structure in which the server pushes a small header of - * the result and lets the client pull the remaining result on demand (at the - * \ref nssi_wait function). - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data. - * @param res_addr The remote memory descriptor for the result. - */ -int xfer_read_encode_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const xfer_read_encode_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NSSI_OK; - log_level debug_level = xfer_debug_level; - - const int len = args->len; - const int seed = args->seed; - const bool validate = args->validate; - int nbytes = len*sizeof(data_t); - - xfer_read_encode_res res; - - log_debug(debug_level, "starting xfer_read_encode_srvr"); - - memset(&res, 0, sizeof(res)); - - /* allocate space for the outgoing buffer */ - res.array.data_array_t_val = (data_t *)malloc(nbytes); - res.array.data_array_t_len = len; - - if (validate) { - xfer_init_data_array(seed, &res.array); - } - - rc = nssi_send_result(caller, request_id, NSSI_OK, &res, res_addr); - - free(res.array.data_array_t_val); - - return rc; -} - -/** - * @brief Emulate a read operation where the bulk data is sent using - * the \ref nssi_put function. - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int xfer_read_rdma_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const xfer_read_rdma_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc; - log_level debug_level = xfer_debug_level; - - const bool validate = args->validate; - const int seed = args->seed; - - int nbytes = args->len*sizeof(data_t); - - /* allocate space for the outgoing buffer */ - data_array_t array; - array.data_array_t_len = args->len; - array.data_array_t_val = (data_t *)malloc(nbytes); - - /* process array (nothing to do) */ - log_debug(debug_level, "starting xfer_read_rdma_srvr"); - - /* if we need to validate the array, it needs to be initialized */ - if (validate) { - xfer_init_data_array(seed, &array); - } - - log_debug(debug_level, "putting data in client buffer"); - - /* now we need to put the data to the client */ - rc = nssi_put_data(caller, array.data_array_t_val, nbytes, data_addr, -1); - if (rc != NSSI_OK) { - log_warn(debug_level, "could not put data to client"); - return rc; - } - -// for (int idx=0;idx 0) { - log_debug(xfer_debug_level, "Starting server threads"); - - xfer_start_server_threads(num_threads, 1000); - - // The main thread will execute unit it receives a "kill" request - rc = nssi_service_start_wfn(&xfer_svc, &xfer_enqueue_rpc_request); - - xfer_cancel_server_threads(); - } - else { - rc = nssi_service_start(&xfer_svc); - if (rc != NSSI_OK) { - log_info(xfer_debug_level, "exited xfer_svc: %s", - nssi_err_str(rc)); - } - } - - - - sleep(5); - - /* shutdown the xfer_svc */ - log_debug(xfer_debug_level, "shutting down service library"); - nssi_service_fini(&xfer_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/examples/xfer-service/xfer_service_args.x b/packages/trios/examples/xfer-service/xfer_service_args.x deleted file mode 100644 index 4565583ba242..000000000000 --- a/packages/trios/examples/xfer-service/xfer_service_args.x +++ /dev/null @@ -1,249 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* -------------------------------------------------------------------------- */ -/** - * @file xfer_xdr.x - * - * @brief Type definitions for a simple rpc service. - * - * @author Ron Oldfield (raoldfi\@cs.sandia.gov). - * $Revision: 342 $. - * $Date: 2005-05-01 23:30:57 -0600 (Sun, 01 May 2005) $. - * - */ - -/** - * @defgroup xfer_example Nessie Data Transfer Example - * - * The data-transfer example demonstrates a simple client - * and server that transfer an array of 16-byte \ref data_t - * data structures from a parallel application to a set of - * servers. We implemented two variations: - * one that transfers the array of data structures - * with the request, and a second method that has each server - * pull the data using the \ref nssi_get_data() function. Although - * this example is fairly simple, it makes a decent benchmark code - * to evaluate overheads of the Nessie transfer protocols and encoding - * schemes. - * -*/ - -/** - * @defgroup xfer_types Nessie Example Types - * @ingroup xfer_example - * - * @{ - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file. */ -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -/** - * @brief Opcodes for the types of transfer operations. - */ -enum xfer_op { - /** Opcode for writing the data through the function arguments. */ - XFER_WRITE_ENCODE_OP = 1, - /** Opcode for the writing the data through the data channel. */ - XFER_WRITE_RDMA_OP, - /** Opcode for reading the data through the result structure. */ - XFER_READ_ENCODE_OP, - /** Opcode for reading the data throught the data channel. */ - XFER_READ_RDMA_OP -}; - -/** - * @brief Opcodes for the MPI transfer tests - */ -enum xfer_mpi_tag { - /** Opcode for blocking send request */ - XFER_MPI_SEND_REQ_TAG = 1001, - XFER_MPI_SEND_ACK_TAG, - XFER_MPI_SEND_DATA_TAG, - /** Opcode for non-blocking send request */ - XFER_MPI_ISEND_REQ_TAG, - XFER_MPI_ISEND_ACK_TAG, - XFER_MPI_ISEND_DATA_TAG, - /** Opcode for blocking recv request */ - XFER_MPI_RECV_REQ_TAG, - XFER_MPI_RECV_ACK_TAG, - XFER_MPI_RECV_DATA_TAG, - /** Opcode for non-blocking recv request */ - XFER_MPI_IRECV_REQ_TAG, - XFER_MPI_IRECV_ACK_TAG, - XFER_MPI_IRECV_DATA_TAG, - /** Opcode for one-sided put request */ - XFER_MPI_PUT_REQ_TAG, - XFER_MPI_PUT_ACK_TAG, - XFER_MPI_PUT_DATA_TAG, - /** Opcode for one-sided get request */ - XFER_MPI_GET_REQ_TAG, - XFER_MPI_GET_ACK_TAG, - XFER_MPI_GET_DATA_TAG, - /** Opcode to quit the server */ - XFER_MPI_FINI_REQ -}; - -/** - * @brief A 16-byte structure that contains an int, float, and double. - * - * This structure contains an int, float, and double as an example - * of a complex structure with multiple types. This will exercise the - * encoding/decoding features of Nessie. - */ -struct data_t { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -}; - -/** - * @brief Array of 16-byte structures that we can send with the request. - * - * Rpcgen will use this definition to define encoding functions to - * encode and decode an array of \ref data_t structures. We will - * use these functions when sending the array with the request. - */ -typedef data_t data_array_t<>; - -/** - * @brief Arguments for the first transfer operation, XFER_WRITE_ENCODE. - * - * The first transfer operation includes the array of \ref data_t - * structures as an argument of the remote operation. This will - * cause the array to be sent to the server as part of the request. - * The client encodes the data before sending it to the server, the server - * decodes the data structure when it arrives. - */ -struct xfer_write_encode_args { - /** The length of the data array. */ - int32_t len; - /** A seed for initializing the array */ - uint32_t seed; - /** A flag to perform validation */ - bool validate; - /** The array of \ref data_t structures, including length. */ - data_array_t array; - -}; - -/** - * @brief Arguments for the second transfer operation, XFER_WRITE_RDMA. - * - * The second transfer operation only needs to send the length - * of the data array. It uses the data argument in the nssi_call_rpc() - * to identify the raw data for the server to fetch. - */ -struct xfer_write_rdma_args { - /** The length of the data array. */ - int32_t len; - /** A seed for initializing the array */ - uint32_t seed; - /** A flag to perform validation */ - bool validate; -}; - - -/** - * @brief Arguments for the third transfer operation, XFER_READ_ENCODE. - * - * The third transfer operation only needs to send the length - * of the data array. It uses the data argument in the nssi_call_rpc() - * to identify the raw data for the server to fetch. - */ -struct xfer_read_encode_args { - /** The length of the data array. */ - int32_t len; - /** A value used to initialize the data */ - uint32_t seed; - /** A flag to perform validation */ - bool validate; -}; - -/** - * @brief Results for the third transfer operation, XFER_READ_ENCODE. - * - * The result of the xfer_read_encode operation includes the - * array of \ref data_t structures. If the size of array is large, - * the address of array will be sent as part of the result and the - * client will fetch array via RDMA. In either case, the structures - * are encoded by the server and decoded by the client. - */ -struct xfer_read_encode_res { - /** The array of \ref data_t structures, including length. */ - data_array_t array; -}; - -/** - * @brief Arguments for the fourth transfer operation, XFER_READ_RDMA. - * - * The xfer_read_rdma operation includes the array of \ref data_t - * structures as an argument of the remote operation. This will - * cause the array to be sent to the server as part of the request. - */ -struct xfer_read_rdma_args { - /** The length of the data array. */ - int32_t len; - /** A seed for initializing the array */ - uint32_t seed; - /** A flag to perform validation */ - bool validate; -}; - - - -/** - * @} - */ diff --git a/packages/trios/examples/xfer-service/xfer_service_test.cpp b/packages/trios/examples/xfer-service/xfer_service_test.cpp deleted file mode 100644 index 5a300eb0b634..000000000000 --- a/packages/trios/examples/xfer-service/xfer_service_test.cpp +++ /dev/null @@ -1,659 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -/* - * xfer_service_test.cpp - * - * Created on: Aug 22, 2011 - * Author: raoldfi - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" -#include "Trios_nssi_debug.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include "xfer_client.h" -#include "xfer_debug.h" -#include "xfer_util.h" - - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -// Prototypes for client and server codes -int xfer_server_main(nssi_rpc_transport transport, int num_threads, MPI_Comm server_comm); -int xfer_client_main (struct xfer_args &args, nssi_service &xfer_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct xfer_args &args, - const char *prefix) -{ - - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url.c_str() << std::endl; - out << prefix << " \turl-file = " << args.url_file << std::endl; - out << prefix << " \ttransport = " << args.transport_name << std::endl; - - if (args.client_flag) { - out << prefix << " \tio-method = " << args.io_method_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tlen = " << args.len << std::endl; - out << prefix << " \tvalidate = " << ((args.validate_flag)?"true":"false") << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - nssi_service xfer_svc; - - int server_index=0; - int rank_in_server=0; - - int transport_index=-1; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct xfer_args args; - - const int num_io_methods = 8; - const int io_method_vals[] = { - XFER_WRITE_ENCODE_SYNC, XFER_WRITE_ENCODE_ASYNC, - XFER_WRITE_RDMA_SYNC, XFER_WRITE_RDMA_ASYNC, - XFER_READ_ENCODE_SYNC, XFER_READ_ENCODE_ASYNC, - XFER_READ_RDMA_SYNC, XFER_READ_RDMA_ASYNC}; - const char * io_method_names[] = { - "write-encode-sync", "write-encode-async", - "write-rdma-sync", "write-rdma-async", - "read-encode-sync", "read-encode-async", - "read-rdma-sync", "read-rdma-async"}; - - const int nssi_transport_list[] = { - NSSI_RPC_PTL, - NSSI_RPC_PTL, - NSSI_RPC_IB, - NSSI_RPC_IB, - NSSI_RPC_GEMINI, - NSSI_RPC_GEMINI, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGQPAMI, - NSSI_RPC_BGQPAMI, - NSSI_RPC_MPI}; - - const int num_nssi_transports = 11; - const int nssi_transport_vals[] = { - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10 - }; - const char * nssi_transport_names[] = { - "portals", - "ptl", - "infiniband", - "ib", - "gemini", - "gni", - "bgpdcmf", - "dcmf", - "bgqpami", - "pami", - "mpi" - }; - - - // Initialize arguments - args.transport=NSSI_DEFAULT_TRANSPORT; - args.len = 1; - args.delay = 1; - args.io_method = XFER_WRITE_RDMA_SYNC; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.num_servers = 1; - args.num_threads = 0; - args.timeout = 500; - args.num_retries = 5; - args.validate_flag = true; - args.kill_server_flag = true; - args.block_distribution = true; - - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("server", "no-server", &args.server_flag, "Run the server" ); - parser.setOption("client", "no-client", &args.client_flag, "Run the client"); - parser.setOption("len", &args.len, "The number of structures in an input buffer"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("server-url-file", &args.url_file, "File that has URL client uses to find server"); - parser.setOption("validate", "no-validate", &args.validate_flag, "Validate the data"); - parser.setOption("num-servers", &args.num_servers, "Number of server processes"); - parser.setOption("num-threads", &args.num_threads, "Number of threads used by each server process"); - parser.setOption("kill-server", "no-kill-server", &args.kill_server_flag, "Kill the server at the end of the experiment"); - parser.setOption("block-distribution", "rr-distribution", &args.block_distribution, - "Use a block distribution scheme to assign clients to servers"); - - // Set an enumeration command line option for the io_method - parser.setOption("io-method", &args.io_method, num_io_methods, io_method_vals, io_method_names, - "I/O Methods for the example: \n" - "\t\t\twrite-encode-sync : Write data through the RPC args, synchronous\n" - "\t\t\twrite-encode-async: Write data through the RPC args - asynchronous\n" - "\t\t\twrite-rdma-sync : Write data using RDMA (server pulls) - synchronous\n" - "\t\t\twrite-rdma-async: Write data using RDMA (server pulls) - asynchronous\n" - "\t\t\tread-encode-sync : Read data through the RPC result - synchronous\n" - "\t\t\tread-encode-async: Read data through the RPC result - asynchronous\n" - "\t\t\tread-rdma-sync : Read data using RDMA (server puts) - synchronous\n" - "\t\t\tread-rdma-async: Read data using RDMA (server puts) - asynchronous"); - - - // Set an enumeration command line option for the NNTI transport - parser.setOption("transport", &transport_index, num_nssi_transports, nssi_transport_vals, nssi_transport_names, - "NSSI transports (not all are available on every platform): \n" - "\t\t\tportals|ptl : Cray or Schutt\n" - "\t\t\tinfiniband|ib : libibverbs\n" - "\t\t\tgemini|gni : Cray libugni (Gemini or Aries)\n" - "\t\t\tbgpdcmf|dcmf : IBM BG/P DCMF\n" - "\t\t\tbgqpami|pami : IBM BG/Q PAMI\n" - "\t\t\tmpi : isend/irecv implementation\n" - ); - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(args.debug_level, "transport_index=%d", transport_index); - if (transport_index > -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - args.io_method_name=std::string(io_method_names[args.io_method]); - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting xfer-service test", rank); - -#ifdef TRIOS_ENABLE_COMMSPLITTER - if (args.transport == NSSI_RPC_MPI) { - MPI_Pcontrol(0); - } -#endif - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - int color = 0; // color=0-->server, color=1-->client - if (args.client_flag && args.server_flag) { - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Put all the servers as the first ranks. - if (rank < args.num_servers) { - color = 0; - log_debug(debug_level, "rank=%d is a server", rank); - } - else { - color = 1; // all others are clients - log_debug(debug_level, "rank=%d is a client", rank); - } - - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - } - else { - if (args.client_flag) { - color=1; - log_debug(debug_level, "rank=%d is a client", rank); - } - else if (args.server_flag) { - color=0; - log_debug(debug_level, "rank=%d is a server", rank); - } - else { - log_error(debug_level, "Must be either a client or a server"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - } - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - log_debug(debug_level, "%d: Finished splitting communicators", rank); - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // If running as both client and server, gather and distribute - // the server URLs to all the clients. - if (args.server_flag && args.client_flag) { - - std::string all_urls; - - // This needs to be a vector of chars, not a string - all_urls.resize(args.num_servers * NSSI_URL_LEN, '\0'); - - // Have servers gather their URLs - if (color == 0) { - assert(args.num_servers == splitsize); // these should be equal - - log_debug(debug_level, "%d: Gathering urls: my_url=%s", rank, my_url.c_str()); - - // gather all urls to rank 0 of the server comm (also rank 0 of MPI_COMM_WORLD) - MPI_Gather(&my_url[0], NSSI_URL_LEN, MPI_CHAR, - &all_urls[0], NSSI_URL_LEN, MPI_CHAR, 0, comm); - } - - // broadcast the full set of server urls to all processes - MPI_Bcast(&all_urls[0], all_urls.size(), MPI_CHAR, 0, MPI_COMM_WORLD); - - log_debug(debug_level, "%d: Bcast urls, urls.size=%d", rank, all_urls.size()); - - if (color == 1) { - - // For block distribution scheme use the utility function (in xfer_util.cpp) - if (args.block_distribution) { - // Use this utility function to calculate the server_index - xfer_block_partition(args.num_servers, splitsize, splitrank, &server_index, &rank_in_server); - } - - // Use a simple round robin distribution scheme - else { - server_index = splitrank % args.num_servers; - rank_in_server = splitrank / args.num_servers; - } - - // Copy the server url out of the list of urls - int offset = server_index * NSSI_URL_LEN; - - args.server_url = all_urls.substr(offset, NSSI_URL_LEN); - - log_debug(debug_level, "client %d assigned to server \"%s\"", splitrank, args.server_url.c_str()); - } - - - log_debug(debug_level, "%d: Finished distributing server urls, server_url=%s", rank, args.server_url.c_str()); - } - - // If running as a client only, have to get the list of servers from the urlfile. - else if (!args.server_flag && args.client_flag){ - - sleep(args.delay); // give server time to get started - - std::vector< std::string > urlbuf; - xfer_read_server_url_file(args.url_file.c_str(), urlbuf, comm); - args.num_servers = urlbuf.size(); - - // For block distribution scheme use the utility function (in xfer_util.cpp) - if (args.block_distribution) { - // Use this utility function to calculate the server_index - xfer_block_partition(args.num_servers, splitsize, splitrank, &server_index, &rank_in_server); - } - - // Use a simple round robin distribution scheme - else { - server_index = splitrank % args.num_servers; - rank_in_server = splitrank / args.num_servers; - } - - args.server_url = urlbuf[server_index]; - log_debug(debug_level, "client %d assigned to server \"%s\"", splitrank, args.server_url.c_str()); - } - - else if (args.server_flag && !args.client_flag) { - args.server_url = my_url; - - if (args.url_file.empty()) { - log_error(debug_level, "Must set --url-file"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - xfer_write_server_url_file(args.url_file.c_str(), my_url.c_str(), comm); - } - - // Set the debug level for the xfer service. - xfer_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - log_debug(debug_level, "%d: server_url=%s", rank, args.server_url.c_str()); - - print_args(out, args, "%"); - - log_debug(debug_level, "server_url=%s", args.server_url.c_str()); - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (color == 0) { - rc = xfer_server_main((nssi_rpc_transport)args.transport, args.num_threads, comm); - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, node 0, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref xfer_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d, url=%s", i, args.server_url.c_str()); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url.c_str(), args.timeout, &xfer_svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(xfer_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - // wait for all the clients to connect - MPI_Barrier(comm); - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(xfer_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&xfer_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - xfer_client_main(args, xfer_svc, comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if ((args.kill_server_flag) && (rank_in_server == 0)) { - log_debug(debug_level, "%d: Halting xfer service", rank); - rc = nssi_kill(&xfer_svc, 0, 5000); - } - rc=nssi_free_service((nssi_rpc_transport)args.transport, &xfer_svc); - if (rc != NSSI_OK) { - log_error(xfer_debug_level, "could not free svc description: %s", - nssi_err_str(rc)); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/examples/xfer-service/xfer_threads.cpp b/packages/trios/examples/xfer-service/xfer_threads.cpp deleted file mode 100644 index 2f981a42f357..000000000000 --- a/packages/trios/examples/xfer-service/xfer_threads.cpp +++ /dev/null @@ -1,183 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * xfer_threads.cpp - * - * Created on: Aug 20, 2012 - * Author: raoldfi - */ - -#include "Trios_nssi_server.h" -#include "Trios_logger.h" - -#include -#include - -#include "xfer_debug.h" -#include "xfer_threads.h" - - - -std::queue pending_reqs; -pthread_cond_t pending_not_empty; -pthread_cond_t pending_not_full; -pthread_mutex_t pending_mutex; - -static volatile bool time_to_exit=false; - -static uint32_t max_num_reqs; -static std::queue consumer_threads; - -/** This is the consumer thread code. - * It pops requests off the pending queue and calls the - * process_rpc_request function. - */ -void *process_pending_reqs(void *arg) -{ - int rc; - nssi_svc_rpc_request *req = NULL; - log_level debug_level = xfer_debug_level; - - intptr_t id = (intptr_t)arg; - - log_info(debug_level, "%d: starting thread", id); - - // do this outside the loop to prevent tons of calls - pthread_mutex_lock(&pending_mutex); - - while (!time_to_exit) { - - if (!pending_reqs.empty()) { - // pull the next request off the queue - req = pending_reqs.front(); - pending_reqs.pop(); - - // signal the producer thread that the the queue is not "full" - pthread_cond_signal(&pending_not_full); - - // unlock the mutex while we process the request - pthread_mutex_unlock(&pending_mutex); - - log_debug(debug_level, "%d: processing request %d", id, req->id); - - rc = nssi_process_rpc_request(req); - if (rc) { - log_error(xfer_debug_level, "%d: Error processing request", id); - return NULL; - } - pthread_mutex_lock(&pending_mutex); - } - else { - // block this thread until the producer thread signals that the queue is not "empty" - pthread_cond_wait(&pending_not_empty, &pending_mutex); - } - } - - pthread_mutex_unlock(&pending_mutex); - - log_info(debug_level, "%d: exiting process_pending_reqs thread", id); - return NULL; -} - - - -void xfer_start_server_threads(const int num_threads, const int max_reqs) -{ - // initialize the condition and mutex variables for the pending queue - pthread_cond_init(&pending_not_empty, NULL); // default attributes - pthread_cond_init(&pending_not_full, NULL); // default attributes - pthread_mutex_init(&pending_mutex, NULL); // default attributes - - max_num_reqs = max_reqs; - - // start the consumer threads - for (int64_t i=0; i= max_num_reqs) { - pthread_cond_wait(&pending_not_full, &pending_mutex); - } - - log_debug(xfer_debug_level, "Adding request %d to the pending queue", req->id); - - // ok to add the request - pending_reqs.push(req); - - // tell the consumer threads the queue has new data - pthread_cond_signal(&pending_not_empty); - - pthread_mutex_unlock(&pending_mutex); - - return(0); -} - -void xfer_cancel_server_threads() -{ - log_debug(xfer_debug_level, "Canceling server threads"); - - pthread_mutex_lock(&pending_mutex); - time_to_exit = true; - pthread_cond_broadcast(&pending_not_empty); - pthread_mutex_unlock(&pending_mutex); - - while (!consumer_threads.empty()) { - pthread_t tid = consumer_threads.front(); - consumer_threads.pop(); - pthread_join(tid, NULL); - } -} - diff --git a/packages/trios/examples/xfer-service/xfer_threads.h b/packages/trios/examples/xfer-service/xfer_threads.h deleted file mode 100644 index b43142f474d5..000000000000 --- a/packages/trios/examples/xfer-service/xfer_threads.h +++ /dev/null @@ -1,60 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * xfer_threads.h - * - * Created on: Aug 20, 2012 - * Author: raoldfi - */ - -#ifndef XFER_THREADS_H_ -#define XFER_THREADS_H_ - -#include "Trios_nssi_server.h" - -void xfer_start_server_threads(const int num_threads, const int max_reqs); -int xfer_enqueue_rpc_request(nssi_svc_rpc_request *req); -void xfer_cancel_server_threads(); - - -#endif /* XFER_THREADS_H_ */ diff --git a/packages/trios/examples/xfer-service/xfer_util.cpp b/packages/trios/examples/xfer-service/xfer_util.cpp deleted file mode 100644 index 9964ec21541d..000000000000 --- a/packages/trios/examples/xfer-service/xfer_util.cpp +++ /dev/null @@ -1,320 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * xfer_util.cpp - * - * Created on: Nov 4, 2011 - * Author: raoldfi - */ - - -#include "Trios_nssi_types_xdr.h" -#include "xfer_service_args.h" -#include "xfer_debug.h" -#include "xfer_util.h" -#include "xfer_client.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - - - -/** - * Output the configuration file with this format - * ------ - * line 1: ns # number of servers - * line 2: server_url_0 - * line 3: server_url_1 - * ... - * line n: server_url_n-1 - */ -int xfer_write_server_url_file( - std::string url_fname, - std::string my_url, - MPI_Comm comm) -{ - int rc = 0; - int rank, np; - log_level debug_level = LOG_UNDEFINED; - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &np); - - // output the config file - if (!url_fname.empty()) { - - // gather the urls from all the servers - std::string urls(NSSI_URL_LEN*np, '\0'); - MPI_Gather(&my_url[0], NSSI_URL_LEN, MPI_CHAR, &urls[0], NSSI_URL_LEN, MPI_CHAR, 0, comm); - - // Write the urls to the config file - if (rank == 0) { - int i; - - std::string tmp_fname = url_fname + ".tmp"; - std::ofstream urlfile (tmp_fname.c_str()); - if (urlfile.is_open()) { - // write the number of servers as the first line - urlfile << np << std::endl; - - for (i=0; i &urlbuf, MPI_Comm comm) -{ - log_level debug_level = xfer_debug_level; - int rank, np; - struct stat sbuf; - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &np); - - // variables for the serverURLs - int numServers = 0; - - // Open the file and get all the server urls - if (rank == 0) { - - while (stat(path, &sbuf)) { log_debug(debug_level, "%s: %s", path, strerror(errno)); sleep(1); } - - // try to open the file - std::ifstream urlFile(path); - if (urlFile.is_open()) { - if (urlFile.good()) { - std::string line; - std::getline(urlFile, line); - numServers = std::atoi(line.c_str()); - } - - urlbuf.resize(numServers, std::string(NSSI_URL_LEN, '\0')); - - for (int i=0; idata_array_t_len; - data_t *buf = array->data_array_t_val; - - /* initialize the random seed */ - //initstate(seed, state, 8); - - for (int i=0; idata_array_t_len != arr2->data_array_t_len) { - log_error(debug_level, "arr1->len=%d, arr2->len=%d", - arr1->data_array_t_len, arr2->data_array_t_len); - return -1; - } - - for (int i=0; i<(int)arr1->data_array_t_len; i++) { - if (arr1->data_array_t_val[i].int_val != arr2->data_array_t_val[i].int_val) { - log_error(debug_level, "val[%d].int_val=%d, val[%d].int_val=%d", - i,arr1->data_array_t_val[i].int_val, - i,arr2->data_array_t_val[i].int_val); - return -1; - } - if (arr1->data_array_t_val[i].float_val != arr2->data_array_t_val[i].float_val) { - log_error(debug_level, "val[%d].float_val=%f, val[%d].float_val=%f", - i,arr1->data_array_t_val[i].float_val, - i,arr2->data_array_t_val[i].float_val); - return -1; - } - if (arr1->data_array_t_val[i].double_val != arr2->data_array_t_val[i].double_val) { - log_error(debug_level, "val[%d].double_val=%g, val[%d].double_val=%g", - i,arr1->data_array_t_val[i].double_val, - i,arr2->data_array_t_val[i].double_val); - return -1; - } - } - - return 0; - -} - - -int xfer_validate_array(const int seed, data_array_t *array) -{ - int rc = 0; - log_level debug_level = xfer_debug_level; - - /* Validate the array that was sent through the args */ - data_array_t tmp_array; - - tmp_array.data_array_t_len = array->data_array_t_len; - tmp_array.data_array_t_val = new data_t[array->data_array_t_len]; - - xfer_init_data_array(seed, &tmp_array); - rc = xfer_compare_data_arrays(&tmp_array, array); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - - delete [] tmp_array.data_array_t_val; - return rc; -} - - -/** - * Partition this val into a bin using a simple linear block-partitioning - * scheme. - * - * Returns the bin assignment and the rank within that bin. - * - */ -int xfer_block_partition( - const int num_bins, - const int num_vals, - const int val, - int *bin, - int *rank) -{ - - int per_bin = num_vals / num_bins; - int num_large = num_vals % num_bins; // number of bins with one extra - - int barrier = num_large * (per_bin + 1); // total number of vals in large bins - - if (val < barrier) { - *bin = val / (per_bin + 1); - *rank = val % (per_bin + 1); - } - - else { - *bin = num_large + (val - barrier)/per_bin; - *rank = (val - barrier) % per_bin; - } - - //std::cout << "num_bins=" << num_bins << ", num_vals=" << num_vals; - //std::cout << ": val=" << val << ", bin=" << bin << ", rank=" << rank << std::endl; - - return 0; -} diff --git a/packages/trios/examples/xfer-service/xfer_util.h b/packages/trios/examples/xfer-service/xfer_util.h deleted file mode 100644 index e8d4cf42d6a9..000000000000 --- a/packages/trios/examples/xfer-service/xfer_util.h +++ /dev/null @@ -1,112 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * xfer_util.h - * - * Created on: May 31, 2012 - * Author: raoldfi - */ - -#ifndef XFER_UTIL_H_ -#define XFER_UTIL_H_ - -#include -#include -#include - -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - - -#include "xfer_service_args.h" - -#ifdef __cplusplus -extern "C" { -#endif - -extern int xfer_write_server_url_file( - std::string url_fname, - std::string my_url, - MPI_Comm comm); - - -extern void xfer_read_server_url_file( - const char *path, - std::vector &urlbuf, - MPI_Comm comm); - -/** - * Initialize an array given a starting seed. - * - * This function is used by both the client and the server to generate - * initialize values in a buffer. - */ -extern void xfer_init_data_array( - const unsigned int seed, - data_array_t *array); - -/** - * Compare two arrays. If they are equal, return 0. - */ -extern int xfer_compare_data_arrays( - const data_array_t *arr1, - const data_array_t *arr2); - - -extern int xfer_validate_array( - const int seed, - data_array_t *array); - -extern int xfer_block_partition( - const int num_bins, - const int num_vals, - const int val, - int *bin, - int *rank); - - -#ifdef __cplusplus -} -#endif - -#endif /* XFER_UTIL_H_ */ diff --git a/packages/trios/libraries/CMakeLists.txt b/packages/trios/libraries/CMakeLists.txt deleted file mode 100644 index 2d5f8f00540b..000000000000 --- a/packages/trios/libraries/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -# We need to make sure at least one of the optional TPLs for data transport -# was enabled. There might be a way to do this in the Dependencies.cmake file, -# but I couldn't figure it out. - -IF (NOT ${PACKAGE_NAME}_HAVE_RDMA_TRANSPORT) - MESSAGE(WARNING "Did not find a supported network protocol. " - "Nessie requires one of the following TPLs to be enabled: " - " Portals, CrayPortals, InfiniBand, PAMI, DCMF, BGPDMA, Gemini") -ENDIF() - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nessie/client) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/nessie/server) diff --git a/packages/trios/libraries/commsplitter/CMakeLists.txt b/packages/trios/libraries/commsplitter/CMakeLists.txt deleted file mode 100644 index 0bba38b65d6e..000000000000 --- a/packages/trios/libraries/commsplitter/CMakeLists.txt +++ /dev/null @@ -1,73 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - - -TRIBITS_SUBPACKAGE(commsplitter) - -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_POSIX_C_SOURCE=200112L") - -SET(HEADERS "") -SET(NONINSTALLHEADERS "") -SET(SOURCES "") - -APPEND_SET(NOINSTALLHEADERS - commsplitter.h -) - -APPEND_SET(SOURCES - init.c - logging.c - wrappers.c -) - -############# libcommsplitter ########################## - -TRIBITS_ADD_LIBRARY( - commsplitter - HEADERS ${HEADERS} - NOINSTALLHEADERS ${NOINSTALLHEADERS} - SOURCES ${SOURCES} - DEPLIBS ${DEPLIBS} -lz -) - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/libraries/commsplitter/Makefile b/packages/trios/libraries/commsplitter/Makefile deleted file mode 100644 index 10d07a162cc8..000000000000 --- a/packages/trios/libraries/commsplitter/Makefile +++ /dev/null @@ -1,77 +0,0 @@ -# -#@HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -#Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -#@HEADER - -#CC = cc -#FC = ftn -CC = mpicc -FC = mpif90 -AR = ar -RM = rm -f -CFLAGS = -O2 -OBJS = init.o logging.o wrappers.o -LIB = libcommsplitter.a -TEST = ctest.exe ftest.exe - -CFLAGS = -O0 -g - -all: $(LIB) $(TEST) - -$(LIB): $(OBJS) - $(AR) rcs $(LIB) $(OBJS) - -test: $(TEST) - @echo "******************************************************************" - @echo "Each app should report that it belongs to a communicator of size 2" - @echo "******************************************************************" - mpirun -n 2 ./ctest.exe : -n 2 ./ftest.exe - -debug: all - -$(TEST): $(OBJS) ftest.f90 ctest.c - $(FC) -g ftest.f90 -o ftest.exe -L. -lcommsplitter -lz - $(CC) -g ctest.c -o ctest.exe -L. -lcommsplitter -lz - -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< - -clean: - $(RM) $(OBJS) $(LIB) $(TEST) *~ diff --git a/packages/trios/libraries/commsplitter/cmake/Dependencies.cmake b/packages/trios/libraries/commsplitter/cmake/Dependencies.cmake deleted file mode 100644 index 5b90a9283f80..000000000000 --- a/packages/trios/libraries/commsplitter/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS HPCToolkit) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/libraries/commsplitter/commsplitter.h b/packages/trios/libraries/commsplitter/commsplitter.h deleted file mode 100644 index 0ae1112870fb..000000000000 --- a/packages/trios/libraries/commsplitter/commsplitter.h +++ /dev/null @@ -1,60 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -typedef struct commsplitter_t { - char *app_pathname; - int debug_level; - - int gsize; - int grank; - - /* pcontrol - default is yes*/ - int enabled; - - MPI_Comm split_comm; -} commsplitter_t; - - -extern commsplitter_t commsplitter_data; - - -void commsplitter_log(char *fmt, ...); diff --git a/packages/trios/libraries/commsplitter/ctest.c b/packages/trios/libraries/commsplitter/ctest.c deleted file mode 100644 index 75f34f53ccd0..000000000000 --- a/packages/trios/libraries/commsplitter/ctest.c +++ /dev/null @@ -1,61 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include -#include -#include - -int main(int argc, char **argv) -{ - int np; - int rank; - char buf[256]; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &np); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - gethostname(buf, 256); - printf("ctest: rank %d of %d executing on %s\n", rank, np, buf); - MPI_Finalize(); - - return(0); -} diff --git a/packages/trios/libraries/commsplitter/ftest.f90 b/packages/trios/libraries/commsplitter/ftest.f90 deleted file mode 100644 index 38ae8870be08..000000000000 --- a/packages/trios/libraries/commsplitter/ftest.f90 +++ /dev/null @@ -1,60 +0,0 @@ -!** -!/@HEADER -!/ ************************************************************************ -!/ -!/ Trios: Trilinos I/O Support -!/ Copyright 2011 Sandia Corporation -!/ -!/ Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -!/ the U.S. Government retains certain rights in this software. -!/ -!/ Redistribution and use in source and binary forms, with or without -!/ modification, are permitted provided that the following conditions are -!/ met: -!/ -!/ 1. Redistributions of source code must retain the above copyright -!/ notice, this list of conditions and the following disclaimer. -!/ -!/ 2. Redistributions in binary form must reproduce the above copyright -!/ notice, this list of conditions and the following disclaimer in the -!/ documentation and/or other materials provided with the distribution. -!/ -!/ 3. Neither the name of the Corporation nor the names of the -!/ contributors may be used to endorse or promote products derived from -!/ this software without specific prior written permission. -!/ -!/ THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -!/ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -!/ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -!/ PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -!/ CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -!/ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -!/ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -!/ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -!/ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -!/ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -!/ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -!/ -!/Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -!/ -!/ ************************************************************************* -!/@HEADER -!*/ - PROGRAM commsplitter_test - - INCLUDE 'mpif.h' - - INTEGER info - INTEGER np - INTEGER rank - CHARACTER*(1024) hostname - INTEGER hostname_len - - CALL MPI_Init(info) - CALL MPI_Comm_size(mpi_comm_world,np,info) - CALL MPI_Comm_rank(mpi_comm_world,rank,info) - CALL MPI_Get_processor_name(hostname, hostname_len, info); - PRINT *,'ftest: rank ',rank,' of ',np,' executing on ', trim(hostname) - CALL MPI_Finalize(info) - - end diff --git a/packages/trios/libraries/commsplitter/init.c b/packages/trios/libraries/commsplitter/init.c deleted file mode 100644 index b365e3b888e9..000000000000 --- a/packages/trios/libraries/commsplitter/init.c +++ /dev/null @@ -1,95 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include -#include -#include -#include - -#include - -#include "commsplitter.h" - - - -commsplitter_t commsplitter_data; - - -void commsplitter_getenv(void) -{ - char *envstr=getenv("COMMSPLITTER_DEBUG_LEVEL"); - if (envstr != NULL) { - commsplitter_data.debug_level=atoi(envstr); - } -} - -void commsplitter_init(char *app_name) -{ - int hostname_len; - char hostname[MPI_MAX_PROCESSOR_NAME]; - - commsplitter_data.enabled = 1; - commsplitter_data.debug_level = 0; - commsplitter_data.split_comm = MPI_COMM_NULL; - - PMPI_Comm_rank(MPI_COMM_WORLD, &commsplitter_data.grank); - PMPI_Comm_size(MPI_COMM_WORLD, &commsplitter_data.gsize); - PMPI_Get_processor_name(hostname, &hostname_len); - - commsplitter_getenv(); - - if (commsplitter_data.grank == 0) { - commsplitter_log("\n"); - commsplitter_log("commsplitter initialized\n"); - commsplitter_log("\n"); - } - - commsplitter_log("app_name is %s\n", app_name); - commsplitter_log("successful init of grank=%d on %s\n", commsplitter_data.grank, hostname); - - return; -} - -void commsplitter_fini(void) -{ - return; -} diff --git a/packages/trios/libraries/commsplitter/logging.c b/packages/trios/libraries/commsplitter/logging.c deleted file mode 100644 index 16ff067f5b43..000000000000 --- a/packages/trios/libraries/commsplitter/logging.c +++ /dev/null @@ -1,65 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include -#include - -#include - -#include "commsplitter.h" - - -void commsplitter_log(char *fmt, ...) -{ - va_list args; - FILE *fp = stdout; - - if (commsplitter_data.debug_level <= 0) { - return; - } - - va_start (args, fmt); - fprintf (fp, "COMMSPLITTER (grank=%d): ", commsplitter_data.grank); - vfprintf (fp, fmt, args); - va_end (args); - fflush (fp); -} diff --git a/packages/trios/libraries/commsplitter/wrappers.c b/packages/trios/libraries/commsplitter/wrappers.c deleted file mode 100644 index 82200ccd8e16..000000000000 --- a/packages/trios/libraries/commsplitter/wrappers.c +++ /dev/null @@ -1,5054 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include "Trios_config.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include "commsplitter.h" - -void commsplitter_init(char *app_name); -void commsplitter_fini(void); - - -#ifdef HAVE_TRIOS_HPCTOOLKIT -#include -#define SAMPLING_IS_ACTIVE() hpctoolkit_sampling_is_active() -#define SAMPLING_STOP() hpctoolkit_sampling_stop() -#define SAMPLING_START() hpctoolkit_sampling_start() -#else -#define SAMPLING_IS_ACTIVE() 0 -#define SAMPLING_STOP() -#define SAMPLING_START() -#endif - - -#define COMMSPLITTER_PATH_MAX 1024 - - -#define SUBSTITUTE_COMM \ -{\ - orig_comm = *comm;\ - MPI_Comm_compare(*comm, MPI_COMM_WORLD, &compare_result);\ - if (compare_result == MPI_IDENT) {\ - *comm = commsplitter_data.split_comm;\ - }\ -} - -#define RESTORE_COMM \ -{\ - *comm = orig_comm;\ -} - - - - -static char *get_exe_name(char *pathname) -{ - char *slash; - - slash=rindex(pathname, '/')+1; - - return(slash); -} - -static char *get_app_pathname_from_proc(void) -{ - int exelen, bufsize=COMMSPLITTER_PATH_MAX; - char *buf = NULL; - - buf = malloc(bufsize); - if (buf == NULL) { - commsplitter_log("unable to allocate space for full executable path.\n"); - PMPI_Abort(MPI_COMM_WORLD, -1); - } - - exelen = readlink("/proc/self/exe", buf, COMMSPLITTER_PATH_MAX); - if (exelen == -1) { - free(buf); - } else { - buf[exelen] = '\0'; - return(buf); - } - - return NULL; -} - -static void get_app_args_from_proc(int *argc, char **argv, int max_args) -{ - int i=0; - char *buf; - FILE *f; - char *arg; - - *argc = 0; - *argv = NULL; - - buf = malloc(COMMSPLITTER_PATH_MAX); - f = fopen("/proc/self/cmdline", "r"); - if (f != NULL) { - if (fread(buf, 1, COMMSPLITTER_PATH_MAX, f) > 0) { - arg = buf; - while(*arg != '\0') { - argv[i] = (char *)malloc(strlen(arg)); - strcpy(argv[i], arg); - arg += strlen(argv[i]) + 1; - i++; - if (i==max_args) { - commsplitter_log("ERROR: too many args. truncating."); - break; - } - } - } - *argc = i; - } - - free(buf); - fclose(f); -} - -static int split_comm_world(void) -{ - int rc = 0; - int enabled_save; - uint32_t color=0; - - if(commsplitter_data.app_pathname == NULL) { - commsplitter_log("Unable to determine application's full pathname. Cannot split MPI_COMM_WORLD.\n"); - PMPI_Abort(MPI_COMM_WORLD, -1); - } else { - color = crc32(0L, Z_NULL, 0); - color = crc32(color, (Bytef *)commsplitter_data.app_pathname, strlen(commsplitter_data.app_pathname)); - commsplitter_log("crc32 for app_pathname(%s) is %lu\n", commsplitter_data.app_pathname, color); - } - - enabled_save = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - rc=PMPI_Comm_split(MPI_COMM_WORLD, color, commsplitter_data.grank, &commsplitter_data.split_comm); - commsplitter_data.enabled = enabled_save; - - return(rc); -} - - -static int commsplitter_MPI_Init(int *argc, char ***argv) -{ - int rc = 0; - int enabled_save; - - enabled_save = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - /* stop hpctoolkit sampling (sometimes causes faults) */ - int sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - - rc = PMPI_Init(argc, argv); - - commsplitter_data.enabled = enabled_save; - - commsplitter_data.app_pathname = get_app_pathname_from_proc(); - commsplitter_init(get_exe_name(commsplitter_data.app_pathname)); - commsplitter_log("app_pathname is %s\n", commsplitter_data.app_pathname); - - rc=split_comm_world(); - - if (sampling) SAMPLING_START(); - - return(rc); -} - -extern int MPI_Init(int *argc, char ***argv) -{ - int rc = 0; - - rc = commsplitter_MPI_Init(argc, argv); - - return(rc); -} - -extern void mpi_init_(int *ierr) -{ - int rc = 0; - int argc; - char **argv; - - argv=(char **)malloc(256*sizeof(char*)); - get_app_args_from_proc(&argc, argv, 256); - - rc = commsplitter_MPI_Init(&argc,(char ***) &argv); - *ierr = rc; - - return; -} - - -static int commsplitter_MPI_Init_thread(int *argc, char ***argv, int required, int *provided) -{ - int rc = 0; - int enabled_save; - - enabled_save = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - int sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc = PMPI_Init_thread(argc, argv, required, provided); - - commsplitter_data.enabled = enabled_save; - - commsplitter_data.app_pathname = get_app_pathname_from_proc(); - commsplitter_init(get_exe_name(commsplitter_data.app_pathname)); - commsplitter_log("app_pathname is %s\n", commsplitter_data.app_pathname); - - rc=split_comm_world(); - if (sampling) SAMPLING_START(); - - return(rc); -} - -extern int MPI_Init_thread(int *argc, char ***argv, int required, int *provided) -{ - int rc = 0; - - rc = commsplitter_MPI_Init_thread(argc, argv, required, provided); - - return(rc); -} - -extern void mpi_init_thread_(int *required, int *provided, int *ierr) -{ - int rc = 0; - int argc; - char **argv; - - argv=(char **)malloc(256*sizeof(char*)); - get_app_args_from_proc(&argc, argv, 256); - - rc = commsplitter_MPI_Init_thread(&argc, (char ***) &argv, *required, provided); - *ierr = rc; - - return; -} - - -static int commsplitter_MPI_Finalize() -{ - int rc = 0; - - commsplitter_fini(); - commsplitter_data.enabled = 0; - commsplitter_log("calling PMPI_Finalize\n"); - - int sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc = PMPI_Finalize(); - if (sampling) SAMPLING_START(); - commsplitter_log("returning from PMPI_Finalize\n"); - - return(rc); -} - -extern int MPI_Finalize(void) -{ - int rc = 0; - - rc = commsplitter_MPI_Finalize(); - - return(rc); -} - -extern void mpi_finalize_(int *ierr) -{ - int rc = 0; - - rc = commsplitter_MPI_Finalize(); - *ierr = rc; - - return; -} - -static int commsplitter_MPI_Pcontrol( - const int flag, - va_list args) -{ - commsplitter_log("MPI_Pcontrol enter: flag=%d\n", flag); - - if (flag == 0) { - if (!commsplitter_data.enabled) { - commsplitter_log("WARN: commsplitter already disabled\n"); - } - commsplitter_data.enabled = 0; - } else { - if (commsplitter_data.enabled) { - commsplitter_log("WARN: commsplitter already enabled\n"); - } - commsplitter_data.enabled = 1; - } - - return MPI_SUCCESS; -} - -int MPI_Pcontrol( - const int flag, - ...) -{ - int rc=0; - va_list args; - - va_start(args, flag); - rc = commsplitter_MPI_Pcontrol(flag, args); - va_end(args); - - return(rc); -} - -int mpi_pcontrol_( - int *flag, - ...) -{ - int rc=0; - va_list args; - - va_start(args, flag); - rc = commsplitter_MPI_Pcontrol(*flag, args); - va_end(args); - - return(rc); -} - - - - -static int commsplitter_MPI_Allgather( - void *sendbuf, - int *sendcount, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcount, - MPI_Datatype *recvtype, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Allgather(sendbuf, *sendcount, *sendtype, recvbuf, *recvcount, *recvtype, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Allgather( - void *sendbuf, - int sendcount, - MPI_Datatype sendtype, - void *recvbuf, - int recvcount, - MPI_Datatype recvtype, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Allgather(sendbuf, &sendcount, &sendtype, recvbuf, &recvcount, &recvtype, &comm); - - return(rc); -} - - -extern void mpi_allgather_( - void *sendbuf, - int *sendcount, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcount, - MPI_Fint *recvtype, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Allgather(sendbuf, sendcount, &c_sendtype, recvbuf, recvcount, &c_recvtype, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Allgatherv( - void *sendbuf, - int *sendcount, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcounts, - int *displs, - MPI_Datatype *recvtype, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Allgatherv(sendbuf, *sendcount, *sendtype, recvbuf, recvcounts, displs, *recvtype, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Allgatherv( - void *sendbuf, - int sendcount, - MPI_Datatype sendtype, - void *recvbuf, - int *recvcounts, - int *displs, - MPI_Datatype recvtype, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Allgatherv(sendbuf, &sendcount, &sendtype, recvbuf, recvcounts, displs, &recvtype, &comm); - - return(rc); -} - - -extern void mpi_allgatherv_( - void *sendbuf, - int *sendcount, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcounts, - int *displs, - MPI_Fint *recvtype, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Allgatherv(sendbuf, sendcount, &c_sendtype, recvbuf, recvcounts, displs, &c_recvtype, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Allreduce( - void *sendbuf, - void *recvbuf, - int *count, - MPI_Datatype *datatype, - MPI_Op *op, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Allreduce(sendbuf, recvbuf, *count, *datatype, *op, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Allreduce( - void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Allreduce(sendbuf, recvbuf, &count, &datatype, &op, &comm); - - return(rc); -} - - -extern void mpi_allreduce_( - void *sendbuf, - void *recvbuf, - int *count, - MPI_Fint *datatype, - MPI_Fint *op, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Op c_op; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_op = MPI_Op_f2c(*op); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Allreduce(sendbuf, recvbuf, count, &c_datatype, &c_op, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Alltoall( - void *sendbuf, - int *sendcount, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcnt, - MPI_Datatype *recvtype, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Alltoall(sendbuf, *sendcount, *sendtype, recvbuf, *recvcnt, *recvtype, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Alltoall( - void *sendbuf, - int sendcount, - MPI_Datatype sendtype, - void *recvbuf, - int recvcnt, - MPI_Datatype recvtype, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Alltoall(sendbuf, &sendcount, &sendtype, recvbuf, &recvcnt, &recvtype, &comm); - - return(rc); -} - - -extern void mpi_alltoall_( - void *sendbuf, - int *sendcount, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcnt, - MPI_Fint *recvtype, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Alltoall(sendbuf, sendcount, &c_sendtype, recvbuf, recvcnt, &c_recvtype, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Alltoallv( - void *sendbuf, - int *sendcnts, - int *sdispls, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcnts, - int *rdispls, - MPI_Datatype *recvtype, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Alltoallv(sendbuf, sendcnts, sdispls, *sendtype, recvbuf, recvcnts, rdispls, *recvtype, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Alltoallv( - void *sendbuf, - int *sendcnts, - int *sdispls, - MPI_Datatype sendtype, - void *recvbuf, - int *recvcnts, - int *rdispls, - MPI_Datatype recvtype, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Alltoallv(sendbuf, sendcnts, sdispls, &sendtype, recvbuf, recvcnts, rdispls, &recvtype, &comm); - - return(rc); -} - - -extern void mpi_alltoallv_( - void *sendbuf, - int *sendcnts, - int *sdispls, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcnts, - int *rdispls, - MPI_Fint *recvtype, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Alltoallv(sendbuf, sendcnts, sdispls, &c_sendtype, recvbuf, recvcnts, rdispls, &c_recvtype, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Attr_delete( - MPI_Comm *comm, - int *keyval) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Attr_delete(*comm, *keyval); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Attr_delete( - MPI_Comm comm, - int keyval) -{ - int rc; - - - rc = commsplitter_MPI_Attr_delete(&comm, &keyval); - - return(rc); -} - - -extern void mpi_attr_delete_( - MPI_Fint *comm, - int *keyval, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Attr_delete(&c_comm, keyval); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Attr_get( - MPI_Comm *comm, - int *keyval, - void *attr_value, - int *flag) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Attr_get(*comm, *keyval, attr_value, flag); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Attr_get( - MPI_Comm comm, - int keyval, - void *attr_value, - int *flag) -{ - int rc; - - - rc = commsplitter_MPI_Attr_get(&comm, &keyval, attr_value, flag); - - return(rc); -} - - -extern void mpi_attr_get_( - MPI_Fint *comm, - int *keyval, - void *attr_value, - int *flag, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Attr_get(&c_comm, keyval, attr_value, flag); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Attr_put( - MPI_Comm *comm, - int *keyval, - void *attr_value) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Attr_put(*comm, *keyval, attr_value); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Attr_put( - MPI_Comm comm, - int keyval, - void *attr_value) -{ - int rc; - - - rc = commsplitter_MPI_Attr_put(&comm, &keyval, attr_value); - - return(rc); -} - - -extern void mpi_attr_put_( - MPI_Fint *comm, - int *keyval, - void *attr_value, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Attr_put(&c_comm, keyval, attr_value); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Barrier( - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Barrier(*comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Barrier( - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Barrier(&comm); - - return(rc); -} - - -extern void mpi_barrier_( - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Barrier(&c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Bcast( - void *buffer, - int *count, - MPI_Datatype *datatype, - int *root, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Bcast(buffer, *count, *datatype, *root, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Bcast( - void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Bcast(buffer, &count, &datatype, &root, &comm); - - return(rc); -} - - -extern void mpi_bcast_( - void *buffer, - int *count, - MPI_Fint *datatype, - int *root, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Bcast(buffer, count, &c_datatype, root, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Bsend( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Bsend(buf, *count, *datatype, *dest, *tag, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Bsend( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Bsend(buf, &count, &datatype, &dest, &tag, &comm); - - return(rc); -} - - -extern void mpi_bsend_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Bsend(buf, count, &c_datatype, dest, tag, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Bsend_init( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Bsend_init(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Bsend_init( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Bsend_init(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_bsend_init_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Bsend_init(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Cart_coords( - MPI_Comm *comm, - int *rank, - int *maxdims, - int *coords) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_coords(*comm, *rank, *maxdims, coords); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_coords( - MPI_Comm comm, - int rank, - int maxdims, - int *coords) -{ - int rc; - - - rc = commsplitter_MPI_Cart_coords(&comm, &rank, &maxdims, coords); - - return(rc); -} - - -extern void mpi_cart_coords_( - MPI_Fint *comm, - int *rank, - int *maxdims, - int *coords, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_coords(&c_comm, rank, maxdims, coords); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Cart_create( - MPI_Comm *comm, - int *ndims, - int *dims, - int *periods, - int *reorder, - MPI_Comm *comm_cart) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_create(*comm, *ndims, dims, periods, *reorder, comm_cart); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_create( - MPI_Comm comm, - int ndims, - int *dims, - int *periods, - int reorder, - MPI_Comm *comm_cart) -{ - int rc; - - - rc = commsplitter_MPI_Cart_create(&comm, &ndims, dims, periods, &reorder, comm_cart); - - return(rc); -} - - -extern void mpi_cart_create_( - MPI_Fint *comm, - int *ndims, - int *dims, - int *periods, - int *reorder, - MPI_Fint *comm_cart, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Comm c_comm_cart; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_create(&c_comm, ndims, dims, periods, reorder, &c_comm_cart); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *comm_cart = MPI_Comm_c2f(c_comm_cart); - } - return; -} - - - - - -static int commsplitter_MPI_Cart_get( - MPI_Comm *comm, - int *maxdims, - int *dims, - int *periods, - int *coords) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_get(*comm, *maxdims, dims, periods, coords); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_get( - MPI_Comm comm, - int maxdims, - int *dims, - int *periods, - int *coords) -{ - int rc; - - - rc = commsplitter_MPI_Cart_get(&comm, &maxdims, dims, periods, coords); - - return(rc); -} - - -extern void mpi_cart_get_( - MPI_Fint *comm, - int *maxdims, - int *dims, - int *periods, - int *coords, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_get(&c_comm, maxdims, dims, periods, coords); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Cart_map( - MPI_Comm *comm, - int *ndims, - int *dims, - int *periods, - int *newrank) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_map(*comm, *ndims, dims, periods, newrank); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_map( - MPI_Comm comm, - int ndims, - int *dims, - int *periods, - int *newrank) -{ - int rc; - - - rc = commsplitter_MPI_Cart_map(&comm, &ndims, dims, periods, newrank); - - return(rc); -} - - -extern void mpi_cart_map_( - MPI_Fint *comm, - int *ndims, - int *dims, - int *periods, - int *newrank, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_map(&c_comm, ndims, dims, periods, newrank); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Cart_rank( - MPI_Comm *comm, - int *coords, - int *rank) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_rank(*comm, coords, rank); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_rank( - MPI_Comm comm, - int *coords, - int *rank) -{ - int rc; - - - rc = commsplitter_MPI_Cart_rank(&comm, coords, rank); - - return(rc); -} - - -extern void mpi_cart_rank_( - MPI_Fint *comm, - int *coords, - int *rank, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_rank(&c_comm, coords, rank); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Cart_shift( - MPI_Comm *comm, - int *direction, - int *displ, - int *source, - int *dest) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_shift(*comm, *direction, *displ, source, dest); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_shift( - MPI_Comm comm, - int direction, - int displ, - int *source, - int *dest) -{ - int rc; - - - rc = commsplitter_MPI_Cart_shift(&comm, &direction, &displ, source, dest); - - return(rc); -} - - -extern void mpi_cart_shift_( - MPI_Fint *comm, - int *direction, - int *displ, - int *source, - int *dest, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_shift(&c_comm, direction, displ, source, dest); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Cart_sub( - MPI_Comm *comm, - int *remain_dims, - MPI_Comm *comm_new) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cart_sub(*comm, remain_dims, comm_new); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cart_sub( - MPI_Comm comm, - int *remain_dims, - MPI_Comm *comm_new) -{ - int rc; - - - rc = commsplitter_MPI_Cart_sub(&comm, remain_dims, comm_new); - - return(rc); -} - - -extern void mpi_cart_sub_( - MPI_Fint *comm, - int *remain_dims, - MPI_Fint *comm_new, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Comm c_comm_new; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cart_sub(&c_comm, remain_dims, &c_comm_new); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *comm_new = MPI_Comm_c2f(c_comm_new); - } - return; -} - - - - - -static int commsplitter_MPI_Cartdim_get( - MPI_Comm *comm, - int *ndims) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Cartdim_get(*comm, ndims); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Cartdim_get( - MPI_Comm comm, - int *ndims) -{ - int rc; - - - rc = commsplitter_MPI_Cartdim_get(&comm, ndims); - - return(rc); -} - - -extern void mpi_cartdim_get_( - MPI_Fint *comm, - int *ndims, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Cartdim_get(&c_comm, ndims); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Comm_create( - MPI_Comm *comm, - MPI_Group *group, - MPI_Comm *comm_out) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_create(*comm, *group, comm_out); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_create( - MPI_Comm comm, - MPI_Group group, - MPI_Comm *comm_out) -{ - int rc; - - - rc = commsplitter_MPI_Comm_create(&comm, &group, comm_out); - - return(rc); -} - - -extern void mpi_comm_create_( - MPI_Fint *comm, - MPI_Fint *group, - MPI_Fint *comm_out, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Group c_group; - MPI_Comm c_comm_out; - - c_comm = MPI_Comm_f2c(*comm); - c_group = MPI_Group_f2c(*group); - - rc = commsplitter_MPI_Comm_create(&c_comm, &c_group, &c_comm_out); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *comm_out = MPI_Comm_c2f(c_comm_out); - } - return; -} - - - - - -static int commsplitter_MPI_Comm_dup( - MPI_Comm *comm, - MPI_Comm *comm_out) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_dup(*comm, comm_out); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_dup( - MPI_Comm comm, - MPI_Comm *comm_out) -{ - int rc; - - - rc = commsplitter_MPI_Comm_dup(&comm, comm_out); - - return(rc); -} - - -extern void mpi_comm_dup_( - MPI_Fint *comm, - MPI_Fint *comm_out, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Comm c_comm_out; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_dup(&c_comm, &c_comm_out); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *comm_out = MPI_Comm_c2f(c_comm_out); - } - return; -} - - - - - -static int commsplitter_MPI_Comm_group( - MPI_Comm *comm, - MPI_Group *group) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_group(*comm, group); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_group( - MPI_Comm comm, - MPI_Group *group) -{ - int rc; - - - rc = commsplitter_MPI_Comm_group(&comm, group); - - return(rc); -} - - -extern void mpi_comm_group_( - MPI_Fint *comm, - MPI_Fint *group, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Group c_group; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_group(&c_comm, &c_group); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *group = MPI_Group_c2f(c_group); - } - return; -} - - - - - -static int commsplitter_MPI_Comm_rank( - MPI_Comm *comm, - int *rank) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_rank(*comm, rank); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_rank( - MPI_Comm comm, - int *rank) -{ - int rc; - - - rc = commsplitter_MPI_Comm_rank(&comm, rank); - - return(rc); -} - - -extern void mpi_comm_rank_( - MPI_Fint *comm, - int *rank, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_rank(&c_comm, rank); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Comm_remote_group( - MPI_Comm *comm, - MPI_Group *group) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_remote_group(*comm, group); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_remote_group( - MPI_Comm comm, - MPI_Group *group) -{ - int rc; - - - rc = commsplitter_MPI_Comm_remote_group(&comm, group); - - return(rc); -} - - -extern void mpi_comm_remote_group_( - MPI_Fint *comm, - MPI_Fint *group, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Group c_group; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_remote_group(&c_comm, &c_group); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *group = MPI_Group_c2f(c_group); - } - return; -} - - - - - -static int commsplitter_MPI_Comm_remote_size( - MPI_Comm *comm, - int *size) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_remote_size(*comm, size); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_remote_size( - MPI_Comm comm, - int *size) -{ - int rc; - - - rc = commsplitter_MPI_Comm_remote_size(&comm, size); - - return(rc); -} - - -extern void mpi_comm_remote_size_( - MPI_Fint *comm, - int *size, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_remote_size(&c_comm, size); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Comm_size( - MPI_Comm *comm, - int *size) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_size(*comm, size); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_size( - MPI_Comm comm, - int *size) -{ - int rc; - - - rc = commsplitter_MPI_Comm_size(&comm, size); - - return(rc); -} - - -extern void mpi_comm_size_( - MPI_Fint *comm, - int *size, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_size(&c_comm, size); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Comm_split( - MPI_Comm *comm, - int *color, - int *key, - MPI_Comm *comm_out) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_split(*comm, *color, *key, comm_out); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_split( - MPI_Comm comm, - int color, - int key, - MPI_Comm *comm_out) -{ - int rc; - - - rc = commsplitter_MPI_Comm_split(&comm, &color, &key, comm_out); - - return(rc); -} - - -extern void mpi_comm_split_( - MPI_Fint *comm, - int *color, - int *key, - MPI_Fint *comm_out, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Comm c_comm_out; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_split(&c_comm, color, key, &c_comm_out); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *comm_out = MPI_Comm_c2f(c_comm_out); - } - return; -} - - - - - -static int commsplitter_MPI_Comm_test_inter( - MPI_Comm *comm, - int *flag) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Comm_test_inter(*comm, flag); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Comm_test_inter( - MPI_Comm comm, - int *flag) -{ - int rc; - - - rc = commsplitter_MPI_Comm_test_inter(&comm, flag); - - return(rc); -} - - -extern void mpi_comm_test_inter_( - MPI_Fint *comm, - int *flag, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Comm_test_inter(&c_comm, flag); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Errhandler_get( - MPI_Comm *comm, - MPI_Errhandler *errhandler) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Errhandler_get(*comm, errhandler); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Errhandler_get( - MPI_Comm comm, - MPI_Errhandler *errhandler) -{ - int rc; - - - rc = commsplitter_MPI_Errhandler_get(&comm, errhandler); - - return(rc); -} - - -extern void mpi_errhandler_get_( - MPI_Fint *comm, - MPI_Errhandler *errhandler, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Errhandler_get(&c_comm, errhandler); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Errhandler_set( - MPI_Comm *comm, - MPI_Errhandler *errhandler) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Errhandler_set(*comm, *errhandler); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Errhandler_set( - MPI_Comm comm, - MPI_Errhandler errhandler) -{ - int rc; - - - rc = commsplitter_MPI_Errhandler_set(&comm, &errhandler); - - return(rc); -} - - -extern void mpi_errhandler_set_( - MPI_Fint *comm, - MPI_Errhandler *errhandler, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Errhandler_set(&c_comm, errhandler); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_File_open( - MPI_Comm *comm, - char *filename, - int *amode, - MPI_Info *info, - MPI_File *fh) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_File_open(*comm, filename, *amode, *info, fh); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_File_open( - MPI_Comm comm, - char *filename, - int amode, - MPI_Info info, - MPI_File *fh) -{ - int rc; - - - rc = commsplitter_MPI_File_open(&comm, filename, &amode, &info, fh); - - return(rc); -} - - -extern void mpi_file_open_( - MPI_Fint *comm, - char *filename, - int *amode, - MPI_Fint *info, - MPI_Fint *fh, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Info c_info; - MPI_File c_fh; - - c_comm = MPI_Comm_f2c(*comm); - c_info = MPI_Info_f2c(*info); - - rc = commsplitter_MPI_File_open(&c_comm, filename, amode, &c_info, &c_fh); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *fh = MPI_File_c2f(c_fh); - } - return; -} - - - - - -static int commsplitter_MPI_Gather( - void *sendbuf, - int *sendcnt, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcount, - MPI_Datatype *recvtype, - int *root, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Gather(sendbuf, *sendcnt, *sendtype, recvbuf, *recvcount, *recvtype, *root, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Gather( - void *sendbuf, - int sendcnt, - MPI_Datatype sendtype, - void *recvbuf, - int recvcount, - MPI_Datatype recvtype, - int root, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Gather(sendbuf, &sendcnt, &sendtype, recvbuf, &recvcount, &recvtype, &root, &comm); - - return(rc); -} - - -extern void mpi_gather_( - void *sendbuf, - int *sendcnt, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcount, - MPI_Fint *recvtype, - int *root, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Gather(sendbuf, sendcnt, &c_sendtype, recvbuf, recvcount, &c_recvtype, root, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Gatherv( - void *sendbuf, - int *sendcnt, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcnts, - int *displs, - MPI_Datatype *recvtype, - int *root, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Gatherv(sendbuf, *sendcnt, *sendtype, recvbuf, recvcnts, displs, *recvtype, *root, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Gatherv( - void *sendbuf, - int sendcnt, - MPI_Datatype sendtype, - void *recvbuf, - int *recvcnts, - int *displs, - MPI_Datatype recvtype, - int root, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Gatherv(sendbuf, &sendcnt, &sendtype, recvbuf, recvcnts, displs, &recvtype, &root, &comm); - - return(rc); -} - - -extern void mpi_gatherv_( - void *sendbuf, - int *sendcnt, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcnts, - int *displs, - MPI_Fint *recvtype, - int *root, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Gatherv(sendbuf, sendcnt, &c_sendtype, recvbuf, recvcnts, displs, &c_recvtype, root, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Graph_create( - MPI_Comm *comm, - int *nnodes, - int *index, - int *edges, - int *reorder, - MPI_Comm *comm_graph) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Graph_create(*comm, *nnodes, index, edges, *reorder, comm_graph); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Graph_create( - MPI_Comm comm, - int nnodes, - int *index, - int *edges, - int reorder, - MPI_Comm *comm_graph) -{ - int rc; - - - rc = commsplitter_MPI_Graph_create(&comm, &nnodes, index, edges, &reorder, comm_graph); - - return(rc); -} - - -extern void mpi_graph_create_( - MPI_Fint *comm, - int *nnodes, - int *index, - int *edges, - int *reorder, - MPI_Fint *comm_graph, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - MPI_Comm c_comm_graph; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Graph_create(&c_comm, nnodes, index, edges, reorder, &c_comm_graph); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *comm_graph = MPI_Comm_c2f(c_comm_graph); - } - return; -} - - - - - -static int commsplitter_MPI_Graph_get( - MPI_Comm *comm, - int *maxindex, - int *maxedges, - int *index, - int *edges) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Graph_get(*comm, *maxindex, *maxedges, index, edges); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Graph_get( - MPI_Comm comm, - int maxindex, - int maxedges, - int *index, - int *edges) -{ - int rc; - - - rc = commsplitter_MPI_Graph_get(&comm, &maxindex, &maxedges, index, edges); - - return(rc); -} - - -extern void mpi_graph_get_( - MPI_Fint *comm, - int *maxindex, - int *maxedges, - int *index, - int *edges, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Graph_get(&c_comm, maxindex, maxedges, index, edges); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Graph_map( - MPI_Comm *comm, - int *nnodes, - int *index, - int *edges, - int *newrank) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Graph_map(*comm, *nnodes, index, edges, newrank); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Graph_map( - MPI_Comm comm, - int nnodes, - int *index, - int *edges, - int *newrank) -{ - int rc; - - - rc = commsplitter_MPI_Graph_map(&comm, &nnodes, index, edges, newrank); - - return(rc); -} - - -extern void mpi_graph_map_( - MPI_Fint *comm, - int *nnodes, - int *index, - int *edges, - int *newrank, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Graph_map(&c_comm, nnodes, index, edges, newrank); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Graph_neighbors( - MPI_Comm *comm, - int *rank, - int *maxneighbors, - int *neighbors) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Graph_neighbors(*comm, *rank, *maxneighbors, neighbors); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Graph_neighbors( - MPI_Comm comm, - int rank, - int maxneighbors, - int *neighbors) -{ - int rc; - - - rc = commsplitter_MPI_Graph_neighbors(&comm, &rank, &maxneighbors, neighbors); - - return(rc); -} - - -extern void mpi_graph_neighbors_( - MPI_Fint *comm, - int *rank, - int *maxneighbors, - int *neighbors, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Graph_neighbors(&c_comm, rank, maxneighbors, neighbors); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Graph_neighbors_count( - MPI_Comm *comm, - int *rank, - int *nneighbors) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Graph_neighbors_count(*comm, *rank, nneighbors); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Graph_neighbors_count( - MPI_Comm comm, - int rank, - int *nneighbors) -{ - int rc; - - - rc = commsplitter_MPI_Graph_neighbors_count(&comm, &rank, nneighbors); - - return(rc); -} - - -extern void mpi_graph_neighbors_count_( - MPI_Fint *comm, - int *rank, - int *nneighbors, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Graph_neighbors_count(&c_comm, rank, nneighbors); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Graphdims_get( - MPI_Comm *comm, - int *nnodes, - int *nedges) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Graphdims_get(*comm, nnodes, nedges); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Graphdims_get( - MPI_Comm comm, - int *nnodes, - int *nedges) -{ - int rc; - - - rc = commsplitter_MPI_Graphdims_get(&comm, nnodes, nedges); - - return(rc); -} - - -extern void mpi_graphdims_get_( - MPI_Fint *comm, - int *nnodes, - int *nedges, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Graphdims_get(&c_comm, nnodes, nedges); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Iprobe( - int *source, - int *tag, - MPI_Comm *comm, - int *flag, - MPI_Status *status) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Iprobe(*source, *tag, *comm, flag, status); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Iprobe( - int source, - int tag, - MPI_Comm comm, - int *flag, - MPI_Status *status) -{ - int rc; - - - rc = commsplitter_MPI_Iprobe(&source, &tag, &comm, flag, status); - - return(rc); -} - - -extern void mpi_iprobe_( - int *source, - int *tag, - MPI_Fint *comm, - int *flag, - MPI_Status *status, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Iprobe(source, tag, &c_comm, flag, status); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Irecv( - void *buf, - int *count, - MPI_Datatype *datatype, - int *source, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Irecv(buf, *count, *datatype, *source, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Irecv( - void *buf, - int count, - MPI_Datatype datatype, - int source, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Irecv(buf, &count, &datatype, &source, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_irecv_( - void *buf, - int *count, - MPI_Fint *datatype, - int *source, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Irecv(buf, count, &c_datatype, source, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Irsend( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Irsend(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Irsend( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Irsend(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_irsend_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Irsend(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Isend( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Isend(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Isend( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Isend(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_isend_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Isend(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Issend( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Issend(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Issend( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Issend(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_issend_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Issend(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Pack( - void *inbuf, - int *incount, - MPI_Datatype *datatype, - void *outbuf, - int *count, - int *position, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Pack(inbuf, *incount, *datatype, outbuf, *count, position, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Pack( - void *inbuf, - int incount, - MPI_Datatype datatype, - void *outbuf, - int count, - int *position, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Pack(inbuf, &incount, &datatype, outbuf, &count, position, &comm); - - return(rc); -} - - -extern void mpi_pack_( - void *inbuf, - int *incount, - MPI_Fint *datatype, - void *outbuf, - int *count, - int *position, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Pack(inbuf, incount, &c_datatype, outbuf, count, position, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Pack_size( - int *incount, - MPI_Datatype *datatype, - MPI_Comm *comm, - int *size) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Pack_size(*incount, *datatype, *comm, size); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Pack_size( - int incount, - MPI_Datatype datatype, - MPI_Comm comm, - int *size) -{ - int rc; - - - rc = commsplitter_MPI_Pack_size(&incount, &datatype, &comm, size); - - return(rc); -} - - -extern void mpi_pack_size_( - int *incount, - MPI_Fint *datatype, - MPI_Fint *comm, - int *size, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Pack_size(incount, &c_datatype, &c_comm, size); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Probe( - int *source, - int *tag, - MPI_Comm *comm, - MPI_Status *status) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Probe(*source, *tag, *comm, status); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Probe( - int source, - int tag, - MPI_Comm comm, - MPI_Status *status) -{ - int rc; - - - rc = commsplitter_MPI_Probe(&source, &tag, &comm, status); - - return(rc); -} - - -extern void mpi_probe_( - int *source, - int *tag, - MPI_Fint *comm, - MPI_Status *status, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Probe(source, tag, &c_comm, status); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Recv( - void *buf, - int *count, - MPI_Datatype *datatype, - int *source, - int *tag, - MPI_Comm *comm, - MPI_Status *status) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Recv(buf, *count, *datatype, *source, *tag, *comm, status); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Recv( - void *buf, - int count, - MPI_Datatype datatype, - int source, - int tag, - MPI_Comm comm, - MPI_Status *status) -{ - int rc; - - - rc = commsplitter_MPI_Recv(buf, &count, &datatype, &source, &tag, &comm, status); - - return(rc); -} - - -extern void mpi_recv_( - void *buf, - int *count, - MPI_Fint *datatype, - int *source, - int *tag, - MPI_Fint *comm, - MPI_Status *status, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Recv(buf, count, &c_datatype, source, tag, &c_comm, status); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Recv_init( - void *buf, - int *count, - MPI_Datatype *datatype, - int *source, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Recv_init(buf, *count, *datatype, *source, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Recv_init( - void *buf, - int count, - MPI_Datatype datatype, - int source, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Recv_init(buf, &count, &datatype, &source, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_recv_init_( - void *buf, - int *count, - MPI_Fint *datatype, - int *source, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Recv_init(buf, count, &c_datatype, source, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Reduce( - void *sendbuf, - void *recvbuf, - int *count, - MPI_Datatype *datatype, - MPI_Op *op, - int *root, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Reduce(sendbuf, recvbuf, *count, *datatype, *op, *root, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Reduce( - void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - int root, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Reduce(sendbuf, recvbuf, &count, &datatype, &op, &root, &comm); - - return(rc); -} - - -extern void mpi_reduce_( - void *sendbuf, - void *recvbuf, - int *count, - MPI_Fint *datatype, - MPI_Fint *op, - int *root, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Op c_op; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_op = MPI_Op_f2c(*op); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Reduce(sendbuf, recvbuf, count, &c_datatype, &c_op, root, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Rsend( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Rsend(buf, *count, *datatype, *dest, *tag, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Rsend( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Rsend(buf, &count, &datatype, &dest, &tag, &comm); - - return(rc); -} - - -extern void mpi_rsend_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Rsend(buf, count, &c_datatype, dest, tag, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Rsend_init( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Rsend_init(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Rsend_init( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Rsend_init(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_rsend_init_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Rsend_init(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Scan( - void *sendbuf, - void *recvbuf, - int *count, - MPI_Datatype *datatype, - MPI_Op *op, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Scan(sendbuf, recvbuf, *count, *datatype, *op, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Scan( - void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Scan(sendbuf, recvbuf, &count, &datatype, &op, &comm); - - return(rc); -} - - -extern void mpi_scan_( - void *sendbuf, - void *recvbuf, - int *count, - MPI_Fint *datatype, - MPI_Fint *op, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Op c_op; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_op = MPI_Op_f2c(*op); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Scan(sendbuf, recvbuf, count, &c_datatype, &c_op, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Scatter( - void *sendbuf, - int *sendcnt, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcnt, - MPI_Datatype *recvtype, - int *root, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Scatter(sendbuf, *sendcnt, *sendtype, recvbuf, *recvcnt, *recvtype, *root, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Scatter( - void *sendbuf, - int sendcnt, - MPI_Datatype sendtype, - void *recvbuf, - int recvcnt, - MPI_Datatype recvtype, - int root, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Scatter(sendbuf, &sendcnt, &sendtype, recvbuf, &recvcnt, &recvtype, &root, &comm); - - return(rc); -} - - -extern void mpi_scatter_( - void *sendbuf, - int *sendcnt, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcnt, - MPI_Fint *recvtype, - int *root, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Scatter(sendbuf, sendcnt, &c_sendtype, recvbuf, recvcnt, &c_recvtype, root, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Scatterv( - void *sendbuf, - int *sendcnts, - int *displs, - MPI_Datatype *sendtype, - void *recvbuf, - int *recvcnt, - MPI_Datatype *recvtype, - int *root, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Scatterv(sendbuf, sendcnts, displs, *sendtype, recvbuf, *recvcnt, *recvtype, *root, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Scatterv( - void *sendbuf, - int *sendcnts, - int *displs, - MPI_Datatype sendtype, - void *recvbuf, - int recvcnt, - MPI_Datatype recvtype, - int root, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Scatterv(sendbuf, sendcnts, displs, &sendtype, recvbuf, &recvcnt, &recvtype, &root, &comm); - - return(rc); -} - - -extern void mpi_scatterv_( - void *sendbuf, - int *sendcnts, - int *displs, - MPI_Fint *sendtype, - void *recvbuf, - int *recvcnt, - MPI_Fint *recvtype, - int *root, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Scatterv(sendbuf, sendcnts, displs, &c_sendtype, recvbuf, recvcnt, &c_recvtype, root, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Send( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Send(buf, *count, *datatype, *dest, *tag, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Send( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Send(buf, &count, &datatype, &dest, &tag, &comm); - - return(rc); -} - - -extern void mpi_send_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Send(buf, count, &c_datatype, dest, tag, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Send_init( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Send_init(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Send_init( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Send_init(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_send_init_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Send_init(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Sendrecv( - void *sendbuf, - int *sendcount, - MPI_Datatype *sendtype, - int *dest, - int *sendtag, - void *recvbuf, - int *recvcount, - MPI_Datatype *recvtype, - int *source, - int *recvtag, - MPI_Comm *comm, - MPI_Status *status) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Sendrecv(sendbuf, *sendcount, *sendtype, *dest, *sendtag, recvbuf, *recvcount, *recvtype, *source, *recvtag, *comm, status); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Sendrecv( - void *sendbuf, - int sendcount, - MPI_Datatype sendtype, - int dest, - int sendtag, - void *recvbuf, - int recvcount, - MPI_Datatype recvtype, - int source, - int recvtag, - MPI_Comm comm, - MPI_Status *status) -{ - int rc; - - - rc = commsplitter_MPI_Sendrecv(sendbuf, &sendcount, &sendtype, &dest, &sendtag, recvbuf, &recvcount, &recvtype, &source, &recvtag, &comm, status); - - return(rc); -} - - -extern void mpi_sendrecv_( - void *sendbuf, - int *sendcount, - MPI_Fint *sendtype, - int *dest, - int *sendtag, - void *recvbuf, - int *recvcount, - MPI_Fint *recvtype, - int *source, - int *recvtag, - MPI_Fint *comm, - MPI_Status *status, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_sendtype; - MPI_Datatype c_recvtype; - MPI_Comm c_comm; - - c_sendtype = MPI_Type_f2c(*sendtype); - c_recvtype = MPI_Type_f2c(*recvtype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Sendrecv(sendbuf, sendcount, &c_sendtype, dest, sendtag, recvbuf, recvcount, &c_recvtype, source, recvtag, &c_comm, status); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Sendrecv_replace( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *sendtag, - int *source, - int *recvtag, - MPI_Comm *comm, - MPI_Status *status) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Sendrecv_replace(buf, *count, *datatype, *dest, *sendtag, *source, *recvtag, *comm, status); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Sendrecv_replace( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int sendtag, - int source, - int recvtag, - MPI_Comm comm, - MPI_Status *status) -{ - int rc; - - - rc = commsplitter_MPI_Sendrecv_replace(buf, &count, &datatype, &dest, &sendtag, &source, &recvtag, &comm, status); - - return(rc); -} - - -extern void mpi_sendrecv_replace_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *sendtag, - int *source, - int *recvtag, - MPI_Fint *comm, - MPI_Status *status, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Sendrecv_replace(buf, count, &c_datatype, dest, sendtag, source, recvtag, &c_comm, status); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Ssend( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Ssend(buf, *count, *datatype, *dest, *tag, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Ssend( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Ssend(buf, &count, &datatype, &dest, &tag, &comm); - - return(rc); -} - - -extern void mpi_ssend_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Ssend(buf, count, &c_datatype, dest, tag, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Ssend_init( - void *buf, - int *count, - MPI_Datatype *datatype, - int *dest, - int *tag, - MPI_Comm *comm, - MPI_Request *request) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Ssend_init(buf, *count, *datatype, *dest, *tag, *comm, request); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Ssend_init( - void *buf, - int count, - MPI_Datatype datatype, - int dest, - int tag, - MPI_Comm comm, - MPI_Request *request) -{ - int rc; - - - rc = commsplitter_MPI_Ssend_init(buf, &count, &datatype, &dest, &tag, &comm, request); - - return(rc); -} - - -extern void mpi_ssend_init_( - void *buf, - int *count, - MPI_Fint *datatype, - int *dest, - int *tag, - MPI_Fint *comm, - MPI_Fint *request, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - MPI_Request c_request; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Ssend_init(buf, count, &c_datatype, dest, tag, &c_comm, &c_request); - - *ierr = (MPI_Fint)rc; - if (rc == MPI_SUCCESS) { - *request = MPI_Request_c2f(c_request); - } - return; -} - - - - - -static int commsplitter_MPI_Topo_test( - MPI_Comm *comm, - int *top_type) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Topo_test(*comm, top_type); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Topo_test( - MPI_Comm comm, - int *top_type) -{ - int rc; - - - rc = commsplitter_MPI_Topo_test(&comm, top_type); - - return(rc); -} - - -extern void mpi_topo_test_( - MPI_Fint *comm, - int *top_type, - MPI_Fint *ierr) -{ - int rc; - - MPI_Comm c_comm; - - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Topo_test(&c_comm, top_type); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Unpack( - void *inbuf, - int *insize, - int *position, - void *outbuf, - int *count, - MPI_Datatype *datatype, - MPI_Comm *comm) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Unpack(inbuf, *insize, position, outbuf, *count, *datatype, *comm); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Unpack( - void *inbuf, - int insize, - int *position, - void *outbuf, - int count, - MPI_Datatype datatype, - MPI_Comm comm) -{ - int rc; - - - rc = commsplitter_MPI_Unpack(inbuf, &insize, position, outbuf, &count, &datatype, &comm); - - return(rc); -} - - -extern void mpi_unpack_( - void *inbuf, - int *insize, - int *position, - void *outbuf, - int *count, - MPI_Fint *datatype, - MPI_Fint *comm, - MPI_Fint *ierr) -{ - int rc; - - MPI_Datatype c_datatype; - MPI_Comm c_comm; - - c_datatype = MPI_Type_f2c(*datatype); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Unpack(inbuf, insize, position, outbuf, count, &c_datatype, &c_comm); - - *ierr = (MPI_Fint)rc; - return; -} - - - - - -static int commsplitter_MPI_Win_create( - void *base, - MPI_Aint *size, - int *disp_unit, - MPI_Info *info, - MPI_Comm *comm, - MPI_Win *win) -{ - int rc, compare_result, enabled_state; - MPI_Comm orig_comm; - - if (commsplitter_data.enabled) { - SUBSTITUTE_COMM; - } - - enabled_state = commsplitter_data.enabled; - commsplitter_data.enabled = 0; - - rc = PMPI_Win_create(base, *size, *disp_unit, *info, *comm, win); - - commsplitter_data.enabled = enabled_state; - if (commsplitter_data.enabled) { - RESTORE_COMM; - } - - return(rc); -} - - - -extern int MPI_Win_create( - void *base, - MPI_Aint size, - int disp_unit, - MPI_Info info, - MPI_Comm comm, - MPI_Win *win) -{ - int rc; - - - rc = commsplitter_MPI_Win_create(base, &size, &disp_unit, &info, &comm, win); - - return(rc); -} - - -extern void mpi_win_create_( - void *base, - MPI_Aint *size, - int *disp_unit, - MPI_Fint *info, - MPI_Fint *comm, - MPI_Win *win, - MPI_Fint *ierr) -{ - int rc; - - MPI_Info c_info; - MPI_Comm c_comm; - - c_info = MPI_Info_f2c(*info); - c_comm = MPI_Comm_f2c(*comm); - - rc = commsplitter_MPI_Win_create(base, size, disp_unit, &c_info, &c_comm, win); - - *ierr = (MPI_Fint)rc; - return; -} diff --git a/packages/trios/libraries/nessie/CMakeLists.txt b/packages/trios/libraries/nessie/CMakeLists.txt deleted file mode 100644 index fabddedf00c5..000000000000 --- a/packages/trios/libraries/nessie/CMakeLists.txt +++ /dev/null @@ -1,44 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - -INCLUDE(TriosProcessXDR) - diff --git a/packages/trios/libraries/nessie/client/Trios_nssi_client.h b/packages/trios/libraries/nessie/client/Trios_nssi_client.h deleted file mode 100644 index 660bd28494b4..000000000000 --- a/packages/trios/libraries/nessie/client/Trios_nssi_client.h +++ /dev/null @@ -1,567 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -/*-------------------------------------------------------------------------*/ -/** - * @file nssi_client.h - * - * @brief Prototypes for the client-side methods for RPC. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * @author Rolf Riesen (rolf\@cs.sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - */ - -#ifndef _NSSI_RPC_CLNT_H_ -#define _NSSI_RPC_CLNT_H_ - -#include "Trios_nssi_request.h" -#include "Trios_nssi_rpc.h" -#include "Trios_nssi_xdr.h" - -/** - * @defgroup rpc_client_api Nessie Client API - * - * An NSSI client communicates with remote services using an - * asynchronous, remote procedure call (RPC) interface. - * As described in @latexonly Section~\ref{sec:Data-movement},@endlatexonly - * this interface uses the Portals message passing API to - * take advantage of special features of the network such - * as RDMA and OS bypass. - * - * The typical usage scenerio includes the following steps: - * - * -# Obtain the service description by calling the - * \ref nssi_lookup_service "\c nssi_lookup_service()" or - * \ref nssi_get_service "\c nssi_get_service()" functions. The service - * description, represented by the ref nssi_service data structure, - * contains details about how to communicate with the - * server, including the remote memory address reserved for incoming - * requests and the encoding scheme to use for control messages. - * -# Call the asynchronous \ref nssi_call_rpc "\c nssi_call_rpc()" function - * specifying the operation and providing buffer space for - * arguments, bulk data, and results. The buffers must remain - * unmodified by the client until the remote operation completes. - * -# To check for completion, call the functions \ref nssi_wait "\c nssi_wait()", - * \ref nssi_timedwait "\c nssi_timedwait()", or \ref nssi_test "\c nssi_test" - * with the \ref nssi_request data structure that was created - * by the \ref nssi_call_rpc "\c nssi_call_rpc()" function. - * - * This remainder of this section contains descriptions of the - * data structures, special data types, and functions that support - * client/service communication. - * - */ - -/** - * @ingroup rpc_client_api - * @brief Register a client stub. - * - * RPC calls require client and server stubs. - * This function registers the client operation and - * assigs the appropriate xdr-encoding functions for - * the arguments, data, and result data structures. - */ -#define NSSI_REGISTER_CLIENT_STUB(opcode, argtype, datatype, rettype) \ - nssi_register_xdr_encoding(opcode, (xdrproc_t)&xdr_ ## argtype, \ - (xdrproc_t)&xdr_ ## datatype, \ - (xdrproc_t)&xdr_ ## rettype) - - -#ifdef __cplusplus -extern "C" { -#endif - - - -#if defined(__STDC__) || defined(__cplusplus) - - - /* - * @brief Initialize an RPC client. - * - * The \ref nssi_rpc_clnt_init function is a blocking call - * that initializes all internal data structures needed by an - * NSSI RPC client. This method is called at least once for each - * remote server that the client expects to use, and the implementation - * of this function may require a communication with that server. - * - * @param server_id The process ID of the remote server. - * @param result Points to a data structure that holds information - * about how to send RPC requests to the remote server. - * - * @return \ref NSSI_OK Indicates success. - * @return \ref NSSI_EBADRPC Indicates an failure in the - * communication library. - * @return \ref NSSI_ETIMEDOUT Indicates that the client timed out - * trying to communicate with the server. - * - */ - extern int nssi_rpc_clnt_init( - NNTI_transport_id_t transport, - const char *url, - nssi_service *result); - - /** - * @brief Initialize an NSSI client. - * - * This function must be called before any other calls to - * the Nessie API. - */ - extern int nssi_init(const nssi_rpc_transport transport_id); - - /** - * @brief Finalize an NSSI client. - * - */ - extern int nssi_fini(const nssi_rpc_transport transport_id); - - /** - * @ingroup rpc_client_api - * - * @brief Get the service descripton from a known host. - * - * The \ref nssi_get_service function contacts a - * remote process to fetch the service description that - * describes how to send and receive RPC requests to the - * service. - * - * @param server_id Identifies the host and process ID of the - * remote service. - * @param timeout The max amount of time (ms) for the operation. Use -1 for infinite. - * @param result Points to the service description of the - * remote service. - * - * @return \ref NSSI_OK Indicates success. - * @return \ref NSSI_EBADRPC Indicates a failure in the communication - * library. - * @return \ref NSSI_ETIMEDOUT Indicates that the client timed out - * trying to communicate with the server. - */ - extern int nssi_get_service( - const nssi_rpc_transport rpc_transport, - const char *url, - const int timeout, - nssi_service *result); - - - /** - * @ingroup rpc_client_api - * - * @brief Get an array of service descriptons. - * - * The \ref nssi_get_services function contacts - * remote processes to fetch service descriptions. - * - * @param server_id Identifies the host and process ID of the - * remote service. - * @param count The number of service descriptors to get. - * @param timeout The max amount of time (ms) for the operation. Use -1 for infinite. - * @param result Points to the service description of the - * remote service. - * - * @return \ref NSSI_OK Indicates success. - * @return \ref NSSI_EBADRPC Indicates a failure in the communication - * library. - * @return \ref NSSI_ETIMEDOUT Indicates that the client timed out - * trying to communicate with the server. - */ - extern int nssi_get_services( - const nssi_rpc_transport rpc_transport, - const char **url, - const int num_servers, - const int timeout, - nssi_service *result); - - extern int nssi_free_service( - const nssi_rpc_transport rpc_transport, - nssi_service *svc); - - /** - * @brief Reset tracing on a designated service. - */ - extern int nssi_trace_reset( - const nssi_service *svc, - const char *trace_fname, - const int trace_ftype, - const char *enable, - const long timeout); - - extern int nssi_kill( - const nssi_service *svc, - const int sig, - const long timeout); - - - - /** - * @brief Setup an RPC request. - * - * @ingroup rpc_ptl_impl - * - * This method populates an RPC request. All input parameters are copied - * into the request. The request can be used to send the RPC to the service - * and wait for the result - * - * @param svc @input descriptor for the NSSI service. - * @param opcode @input descriptor for the remote method. - * @param args @input pointer to the arguments. - * @param data @input pointer to data (for bulk data transfers). - * @param data_size @input length of data buffer - * @param result @input where to put results. - * @param request @output The request handle (used to test for - * completion). - */ - extern int nssi_create_request( - const nssi_service *svc, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *result, - nssi_request *request); - - /** - * @brief Send an RPC request to an NSSI server. - * - * @ingroup rpc_ptl_impl - * - * This method encodes and transfers an RPC request header and - * operation arguments to an NSSI server using NNTI. If the - * arguments are sufficiently small, \b nssi_send_request sends - * the request header and the arguments in a single message. - * If the arguments are large (i.e., too large for the request buffer), - * the server fetches the arguments from a client-side buffer. - * - * @param request @inout The request handle (used to test for completion). - */ - extern int nssi_send_request( - nssi_request *request); - - /** - * @brief Call a remote procedure. - * - * @ingroup rpc_client_api - * - * The nssi_call_rpc() function encodes and - * transfers an RPC request to an NSSI server. It returns - * an integer value that corresponds to a return code - * defined in the enumerator \ref nssi_return_codes; - * however, since the function does not wait for completion - * of the remote method, the return value only indicates - * success or failure of marshaling and tranferring - * the request to the server, not the success of the - * remote method. - * - * The arguments include the function-specific set of arguments - * required by the remote function, optional data and result - * arguments, and the \ref nssi_request data - * structure that holds all required information about the - * pending request. - * - * The ``data'' argument is reserved for functions that perform - * bulk data transfers between the client and the server. - * This argument points to client-side memory buffer - * that the server pulls from (e.g., for reads) or puts - * into (e.g., for writes). This buffer must remain - * unmodified by the client until the remote function completes. - * - * The ``result'' argument is reserved for functions that - * expect a non-integer ``control'' response from the server. - * The type of the result argument is function specific and points to the - * client-side buffer reserved for that data. At first glance, - * this may seem similar to the data argument; however, there - * are several important distinctions: - * - * -# Results are always directed from the server to the client, - * but data could flow in either direction. - * -# Data is typically large in comparison to the ``control'' structures - * (arguments and results). A clever implementation may transfer - * args and results over a network channel optimized for small - * messages, but transfer data over a network channel optimized - * for bulk transfers. - * -# Results are encoded in a portable binary format before transfer, - * data is transferred as ``raw'' binary data. - * - * The final argument is a pointer to the \ref nssi_request structure that - * contains all required information about the pending request, including - * the \ref nssi_request_status "status" of the pending - * request and the return code of the remote method (if complete). - * The client calls the functions nssi_get_status(), - * nssi_test(), or nssi_wait() - * to asynchronously get the status of a pending request, test for - * completion, or wait for completion of a request. - * - * @param svc Points to the data structure that describes how - * to communicate with the remote service. - * @param opcode Identifies the remote operation to execute. - * @param args Points to the unencoded arguments for the remote request. - * @param data In not null, points to memory reserved for bulk data - * this buffer must remain available while the request - * is pending. - * @param data_len The length of the data buffer (if not null). - * @param result Points to a memory reserved for result. This buffer must - * remain available until the request completes. - * @param req Points to a data structure that holds information about - * the pending request. - * - * @return \ref NSSI_OK Indicates success of encoding and transferring the request. - * @return \ref NSSI_EENCODE Indicates an error encoding the request. - * @return \ref NSSI_EBADRPC Indicates an error in the underlying transport library - * while transferring the request. - * @return \ref NSSI_ENOTSUP Indicates an unsupported transport mechanism. - */ - extern int nssi_call_rpc( - const nssi_service *svc, - const int opcode, - void *args, - void *data, - uint32_t data_len, - void *result, - nssi_request *request); - - - /** - * @brief Blocking remote rpc call. - * - * - */ - extern int nssi_call_rpc_sync( - const nssi_service *svc, - const int opcode, - void *args, - void *data, - uint32_t data_len, - void *result); - - /** - * @brief Test for completion of an RPC request. - * - * @ingroup rpc_client_api - * - * The \ref nssi_test function checks the status of the specified - * request and returns TRUE if the status is equal to \ref NSSI_REQUEST_COMPLETE - * or \ref NSSI_REQUEST_ERROR . - * - * @param req Points to the data structure that holds information - * about the request. - * @param rc If the request is complete, points to the return code - * of the completed request. Undefined otherwise. - * - * @return TRUE Indicates that the request is complete. - * @return FALSE Indicates that the request is not complete. - */ - extern int nssi_test( - nssi_request *req, - int *rc); - - /** - * @brief Wait for a fixed amount of time for an RPC request to complete. - * - * @ingroup rpc_client_api - * - * The \ref nssi_timedwait function blocks for no more than - * \em timeout milliseconds waiting for the remote procedure to complete. - * - * @param req Points to the request structure associated with the - * remote function. - * @param timeout The maximum amount of time (milliseconds) that the - * function will block waiting for the remote function - * to complete. - * @param remote_rc If the remote function completes, this parameter is - * the return code of the remote method. Undefined otherwise. - * - * @return \ref NSSI_OK Indicates that the remote function completed - * (possibly with an error). - * @return \ref NSSI_EBADRPC Indicates failure in the low-level transport mechanism. - * @return \ref NSSI_ETIMEDOUT Indicates that the remote procedure failed to complete - * within the alloted time. - */ - extern int nssi_timedwait( - nssi_request *req, - int timeout, - int *remote_rc); - - /** - * @brief Wait for an RPC request to complete. - * - * @ingroup rpc_client_api - * - * The \ref nssi_wait function blocks until the remote procedure - * associated with an RPC request completes. - * - * @param req Points to the request structure associated with the - * remote function. - * @param rc If the remote function completes, this parameter is - * the return code of the remote method. Undefined otherwise. - * - * @return \ref NSSI_OK Indicates that the remote function is complete - * (possibly with an error). - * @return \ref NSSI_EBADRPC Indicates failure in the low-level transport mechanism. - */ - extern int nssi_wait( - nssi_request *req, - int *rc); - - - /** - * @brief Wait for any request to complete. - * - * @ingroup rpc_client_api - * - * The \ref nssi_waitany function blocks for no more than - * \em timeout milliseconds waiting for any one of an array of requests - * to complete. - * - * @param req_array Points to an array of requests. - * @param req_count The size of the request array. - * @param timeout The maximum amount of time (milliseconds) that the - * function will block waiting for a request to complete. - * @param which The index of the complete request. - * @param remote_rc The return code of the completed request. - * - * @return \ref NSSI_OK Indicates that a request completed - * (possibly with an error). - * @return \ref NSSI_EBADRPC Indicates failure in the low-level transport mechanism. - * @return \ref NSSI_ETIMEDOUT Indicates that no request completed - * within the alloted time. - */ - extern int nssi_waitany( - nssi_request *req_array, - nssi_size req_count, - int timeout, - int *which, - int *remote_rc); - - - /** - * @brief Wait for all requests to complete. - * - * A request is not complete unless we receive the short - * result. - * - * @param req_array The array of pending requests. - * @param req_count The number of pending requests. - * @param timeout The time to wait for any one request. - * - */ - extern int nssi_waitall( - nssi_request *req_array, - nssi_size req_count, - int timeout); - - - /** - * @brief Return the status of an RPC request. - * - * @ingroup rpc_client_api - * - * The \ref nssi_get_status function returns the status (if known) - * of a remote request. This function is primarily used for - * debugging. - * - * @param req Points to the request structure associated with the - * remote function. - * @param status The state of the pending request. - * @param rc If the status of the remote function is complete, - * this parameter holds the return code of the remote - * method. Undefined otherwise. - * - * @return \ref NSSI_OK Indicates success. - * @return \ref NSSI_EBADRPC Indicates failure to get the status. - */ - extern int nssi_get_status( - nssi_request *req, - int *status, - int *rc); - - extern int nssi_multicast_rpc( - const nssi_service *svcs, - const nssi_size num_svcs, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *results, - nssi_size result_size, /* size in bytes of the result */ - nssi_request *requests); - - extern int nssi_multicast_rpc_sync( - const nssi_service *svcs, - const nssi_size num_svcs, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *results, - nssi_size result_size); - - - extern int nssi_atomic_increment( - const nssi_service *svc, - const uint64_t remote_atomic, - const uint64_t local_atomic); - - extern int nssi_atomic_decrement( - const nssi_service *svc, - const uint64_t remote_atomic, - const uint64_t local_atomic); - - extern int nssi_atomic_read( - const nssi_service *svc, - const uint64_t local_atomic, - int64_t *value); - - -#else /* K&R C */ -#endif - - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/packages/trios/libraries/nessie/client/Trios_nssi_request.h b/packages/trios/libraries/nessie/client/Trios_nssi_request.h deleted file mode 100644 index f41a0515acc5..000000000000 --- a/packages/trios/libraries/nessie/client/Trios_nssi_request.h +++ /dev/null @@ -1,217 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file nssi_request.h - * - * @brief Definition of the nssi_request data structure. - * - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - */ - -#ifndef _NSSI_REQUEST_H -#define _NSSI_REQUEST_H - - -struct nssi_request; - -#include "Trios_nssi_rpc.h" - -#ifdef __cplusplus -extern "C" { -#endif - - /** - * @ingroup base_types - * - * @brief States for a pending RPC request. - * - * The \ref nssi_request_status enumerator provides - * integer values that identify the state of a pending NSSI request. - */ - enum nssi_request_status { - /** @brief The request is complete with an error. */ - NSSI_REQUEST_ERROR = -1, - - /** @brief The request has no/null status. */ - NSSI_REQUEST_NULL = 0, - - /** @brief The client is sending the request to server. */ - NSSI_SENDING_REQUEST = 1, - - /** @brief The remote server is processing the request. */ - NSSI_PROCESSING_REQUEST, - - /** @brief The client is processing the result. */ - NSSI_PROCESSING_RESULT, - - /** @brief The last wait operation timed out. */ - NSSI_REQUEST_TIMEDOUT, - - /** @brief The request is complete. */ - NSSI_REQUEST_COMPLETE - }; - - typedef enum nssi_request_status nssi_request_status; - - /** - * @ingroup base_types - * - * @brief The request structure. - * - * The \ref nssi_request structure represents a pending NSSI - * request. It contains a unique identifier and pointers to all - * data structures and buffers needed to send the request to the - * remote server and to process the result when the server completes. - * - * \todo We need to abstract out the implementation-specific portions - * of this data structure. These include the fields to encode/decode - * arguments and the all of the Portals data structures. - */ - struct nssi_request { - /** @brief The remote service that is handling this request. */ - const nssi_service *svc; - - int job_type; - - /** @brief An ID for this request. */ - unsigned long id; - - /** @brief The opcode of the remote function. */ - int opcode; - - /** @brief If TRUE, nssi_send_request() will mark this request complete - long args, bulk data and short result are not allowed. */ - int8_t is_responseless; - - /** @brief If TRUE, nssi_send_request() will wait for this request to complete. */ - int8_t is_sync; - - /** @brief If is_sync==TRUE, the timeout for nssi_timedwait(). */ - int8_t sync_timeout; - - /** @brief The error code of request. This value will be \ref NSSI_OK unless the - * request status=\ref NSSI_REQUEST_ERROR . */ - int error_code; - - /** @brief Status of the pending request */ - nssi_request_status status; - - - /*********************** - ** Short Request - ***********************/ - /** @brief If TRUE, the app pinned the short request buffer, so do not unpin/free at cleanup. */ - int8_t app_pinned_short_request; - /** @brief Handle for the buffer where the short request will be put. */ - NNTI_buffer_t *short_request_hdl; - NNTI_buffer_t short_request; - /** @brief Work request to track the short request transfer. */ - NNTI_work_request_t short_request_wr; - - /*********************** - ** Long Args - ***********************/ - /** @brief If TRUE, the app pinned the long args buffer, so do not unpin/free at cleanup. */ - int8_t app_pinned_long_args; - int use_long_args; - /** @brief Points to the memory reserved for the args. */ - void *args; - /** @brief Points to the XDR function used to encode arguments. */ - xdrproc_t xdr_encode_args; - /** @brief Handle for the buffer where the long arguments reside. */ - NNTI_buffer_t *long_args_hdl; - NNTI_buffer_t long_args; - /** @brief Work request to track the long arguments transfer. */ - NNTI_work_request_t long_args_wr; - - /*********************** - ** Bulk Data - ***********************/ - /** @brief If TRUE, the app pinned the bulk data buffer, so do not unpin/free at cleanup. */ - int8_t app_pinned_bulk_data; - /** @brief Points to the memory reserved for the bulk data transfers (NULL if not used). */ - void *data; - /** @brief The max size of the memory used for bulk transfers. */ - uint32_t data_size; - /** @brief Points to the XDR function used to encode data. */ - xdrproc_t xdr_encode_data; - /** @brief Handle for the buffer where the data reside. */ - NNTI_buffer_t *bulk_data_hdl; - NNTI_buffer_t bulk_data; - /** @brief Work request to track the bulk data transfer. */ - NNTI_work_request_t bulk_data_wr; - - /*********************** - ** Short Result - ***********************/ - /** @brief If TRUE, the app pinned the short result buffer, so do not unpin/free at cleanup. */ - int8_t app_pinned_short_result; - /** @brief Points to the memory reserved for the result. */ - void *result; - /** @brief Points to the XDR function used to decode the result. */ - xdrproc_t xdr_decode_result; - /** @brief Handle for the buffer where the short result will be put. */ - NNTI_buffer_t *short_result_hdl; - NNTI_buffer_t short_result; - /** @brief Work request to track the short result transfer. */ - NNTI_work_request_t short_result_wr; - - /*********************** - ** Callback - ***********************/ - /** @brief A callback function used by the wait() function when a request is complete. */ - int (*callback)(struct nssi_request *); - /** @brief Optional arguments for the callback function. */ - void *callback_args; - - }; - - typedef struct nssi_request nssi_request; - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/packages/trios/libraries/nessie/client/nssi_client.cpp b/packages/trios/libraries/nessie/client/nssi_client.cpp deleted file mode 100644 index 12211f58679f..000000000000 --- a/packages/trios/libraries/nessie/client/nssi_client.cpp +++ /dev/null @@ -1,2424 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file rpc_client.cc - * - * @brief Implementation of the \ref rpc_client_api "RPC Client API". - * for the NSSI. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - * - */ -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_request.h" -#include "Trios_nssi_types.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" -#include "Trios_logger.h" -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "buffer_queue.h" -#include "Trios_nssi_rpc.h" -#include "Trios_nssi_xdr.h" -#include "Trios_nssi_debug.h" - -#include -#include -#include -#include -#include - -#include - -#ifdef HAVE_TRIOS_MALLOC_H -#include -#endif - -#include -#include -#include -#include - - -#include "nssi_opcodes.h" -#include "nssi_service_args.h" - - -#include "Trios_nnti.h" - -#define MIN_TIMEOUT 1000 - -#define MAX_RETRIES 50 - -extern NNTI_transport_t transports[NSSI_RPC_COUNT]; -extern nssi_config_t nssi_config; - -static bool client_initialized = false; - -static nthread_counter_t request_count; - -extern trios_buffer_queue_t send_bq; -extern trios_buffer_queue_t recv_bq; -extern trios_buffer_queue_t rdma_target_bq; -extern trios_buffer_queue_t rdma_get_bq; -extern trios_buffer_queue_t rdma_put_bq; - -/** - * @addtogroup rpc_ptl_impl - * - * This section describes a portals implementation of the - * NSSI asynchrounous \ref rpc_api "RPC" mechanism. - */ - -/*--------------------------- Private methods ------------------- */ - -#ifdef ANSI_FUNC - -static int client_init(void) -#else - -static int -client_init () -#endif -{ - int rc = NSSI_OK; - - if (client_initialized) { - return rc; - } - - // Initialize the thread counter - nthread_counter_init(&request_count); - - NSSI_REGISTER_CLIENT_STUB(NSSI_OP_GET_SERVICE, void, void, nssi_service); - NSSI_REGISTER_CLIENT_STUB(NSSI_OP_KILL_SERVICE, nssi_kill_service_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NSSI_OP_TRACE_RESET, nssi_trace_reset_args, void, void); - - client_initialized = true; - - return rc; -} - -/** - * @brief Clean up portals data structures associated with - * operation requests. - * - * @ingroup rpc_ptl_impl - * - * This method frees memory and data structures created in - * the prepare_request and prepare_result methods. - * - * - * If the arguments were too large for the short request, the - * process_request method allocated memory for the encoded - * arguments and all portals data structures for the arguments. - * - * For results, the process_request method allocates space for - * the encoded short result and portals data structures associated - * with the short result. - */ -static int cleanup(nssi_request *request) -{ - return NSSI_OK; -} - - -/** - * @brief Process the result of an operation request. - * - * @ingroup rpc_ptl_impl - * - * After processing an operation request, a NSSI server - * sends a short result back to the client. If the - * contents of the result do not fit into the short result - * the server stores the result until the client can fetch - * it. - * - * This client-side method executes after the short - * result arrives at the client. It decodes the - * result header, possibly fetches the results from - * the server, and decodes the result. - * - * @param encoded_short_res @input The result buffer received from the server. - * @param request @input The request data structure. - */ -static int process_result(char *encoded_short_res_buf, nssi_request *request) -{ - - int rc = NSSI_OK; /* return code */ - uint32_t result_size; - nssi_result_header header; - void *decoded_result = NULL; - XDR hdr_xdrs; - XDR res_xdrs; - - trios_declare_timer(call_time); - - NNTI_buffer_t encoded_long_res_hdl; - NNTI_work_request_t encoded_long_res_wr; - NNTI_status_t status; - - NNTI_buffer_t long_res_ack_hdl; - NNTI_work_request_t long_res_ack_wr; - NNTI_status_t long_res_ack_status; - - log_level debug_level = rpc_debug_level; - - /* assign the result decoding function */ - xdrproc_t xdr_decode_result = request->xdr_decode_result; - - log_debug(debug_level,"Start processing result, status=%d", request->status); - - /** set the status to processing */ - request->status = NSSI_PROCESSING_RESULT; - - /* initialize the header */ - memset(&header, 0, sizeof(nssi_result_header)); - - xdrmem_create(&hdr_xdrs, - encoded_short_res_buf, - NSSI_SHORT_RESULT_SIZE, - XDR_DECODE); - - /* decode the header */ - log_debug(debug_level,"decoding result header..."); - - trios_start_timer(call_time); - if (! xdr_nssi_result_header(&hdr_xdrs, &header)) { - log_fatal(rpc_debug_level,"failed to decode the result header"); - rc = NSSI_EDECODE; - goto cleanup; - } - trios_stop_timer("xdr_nssi_result_header - decode", call_time); - - if (logging_debug(debug_level)) { - fprint_nssi_result_header(logger_get_file(), "header", "DEBUG", &header); - } - - /* what to do if the remote code had an error */ - if (header.rc != NSSI_OK) { - request->status = NSSI_REQUEST_ERROR; - request->error_code = header.rc; - rc = NSSI_OK; - goto cleanup; - } - - /* get result size from the header */ - result_size = header.result_size; - - if (result_size > 0) { - - /* decode the result */ - log_debug(debug_level,"getting the result (size == %d)...", result_size); - decoded_result = request->result; - - /* extract result from header */ - if (!header.fetch_result) { - log_debug(debug_level,"decoding the result..."); - trios_start_timer(call_time); - if (!xdr_decode_result(&hdr_xdrs, decoded_result)) { - log_fatal(debug_level,"failed to decode the result"); - rc = NSSI_EDECODE; - goto cleanup; - } - trios_stop_timer("xdr_decode_result - decode", call_time); - } - - /* else fetch the result from the server */ - else { - log_debug(debug_level,"fetching result (%d bytes) " - "from server...", result_size); - - /* allocate space for result */ - trios_start_timer(call_time); - rc=NNTI_alloc( - &transports[request->svc->transport_id], - result_size, - 1, - NNTI_GET_DST, - &encoded_long_res_hdl); - trios_stop_timer("NNTI_register_memory - long result", call_time); - if (rc != NNTI_OK) { - log_error(debug_level, "failed registering long result: %s", - nnti_err_str(rc)); - } - - /* fetch the data from the server */ - trios_start_timer(call_time); - rc=NNTI_get(&header.result_addr, - 0, - result_size, - &encoded_long_res_hdl, - 0, - &encoded_long_res_wr); - trios_stop_timer("NNTI_get - long result", call_time); - if (rc != NNTI_OK) { - log_error(debug_level, "failed getting long result: %s", - nnti_err_str(rc)); - } - trios_start_timer(call_time); - rc=NNTI_wait(&encoded_long_res_wr, - -1, - &status); - trios_stop_timer("NNTI_wait - long result", call_time); - if (rc != NNTI_OK) { - log_error(debug_level, "failed waiting for the long result: %s", - nnti_err_str(rc)); - } - /* create a memory stream for XDR-decoding the result */ - xdrmem_create(&res_xdrs, NNTI_BUFFER_C_POINTER(&encoded_long_res_hdl), - NNTI_BUFFER_SIZE(&encoded_long_res_hdl), XDR_DECODE); - - log_debug(debug_level,"decoding the fetched result..."); - trios_start_timer(call_time); - if (!xdr_decode_result(&res_xdrs, decoded_result)) { - log_fatal(debug_level,"failed to decode the result"); - rc = NSSI_EDECODE; - goto cleanup; - } - trios_stop_timer("xdr_decode_result - decode", call_time); - - trios_start_timer(call_time); - rc=NNTI_alloc( - &transports[request->svc->transport_id], - sizeof(int8_t), - 1, - NNTI_SEND_SRC, - &long_res_ack_hdl); - trios_stop_timer("NNTI_register_memory - long result ack", call_time); - if (rc != NNTI_OK) { - log_error(debug_level, "failed registering long result ack: %s", - nnti_err_str(rc)); - } - - trios_start_timer(call_time); - rc=NNTI_send( - &request->svc->svc_host, - &long_res_ack_hdl, - &header.result_ack_addr, - &long_res_ack_wr); - trios_stop_timer("NNTI_send - long result ack", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed sending short result: %s", - nnti_err_str(rc)); - } - trios_start_timer(call_time); - rc=NNTI_wait( - &long_res_ack_wr, - -1, - &long_res_ack_status); - trios_stop_timer("NNTI_wait - long result ack", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for short result: %s", - nnti_err_str(rc)); - } - } - } - - request->status = NSSI_REQUEST_COMPLETE; - -cleanup: - if (header.fetch_result) { - NNTI_free(&encoded_long_res_hdl); - if (rc != NNTI_OK) { - log_error(debug_level, "failed unregistering long result: %s", - nnti_err_str(rc)); - } - - NNTI_free(&long_res_ack_hdl); - if (rc != NNTI_OK) { - log_error(debug_level, "failed unregistering long result ack: %s", - nnti_err_str(rc)); - } - } - - /* clean up portals data structures */ - log_debug(debug_level,"clean up data structures"); - cleanup(request); - - log_debug(debug_level,"end"); - - return rc; -} - - -/** - * @brief Initialize an RPC client and return the - * service provided by the server. */ -int nssi_rpc_clnt_init( - const nssi_rpc_transport rpc_transport, - const char *my_url, - const char *svc_url, - nssi_service *result) -{ - int rc = NSSI_OK; - int default_timeout = 1000; - - /* initialize RPC */ - rc = nssi_rpc_init(rpc_transport, NSSI_DEFAULT_ENCODE, my_url); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "could not initialize RPC"); - return rc; - } - - nssi_init(rpc_transport); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "could not initialize nssi client"); - return rc; - } - - /* ping the server */ - return nssi_get_service(rpc_transport, svc_url, default_timeout, result); -} - -/** - * @brief Send a ping request to the server. - */ -int nssi_get_services( - const nssi_rpc_transport rpc_transport, - const char **url, - const int num_servers, - const int timeout, - nssi_service *result) -{ - int rc = NSSI_OK; - int i; - - for (i=0; isvc_host); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed disconnecting from service: %s", - nnti_err_str(rc)); - } - return(rc); -} - -/* --------------- CLIENT INTERFACE --------- */ - -/** - * @brief Initialize the client. - */ -int nssi_init(const nssi_rpc_transport transport_id) -{ - int rc = NSSI_OK; - - client_init(); - - client_initialized=true; - - return rc; -} - -/** - * @brief Complete a Nessie client. - */ -int nssi_fini(const nssi_rpc_transport transport_id) -{ - int rc = NSSI_OK; - - nthread_counter_fini(&request_count); - - client_initialized=false; - - return(rc); -} - -/** - * @brief Send a kill request to the server. - */ -int nssi_kill( - const nssi_service *svc, - const int sig, - const long timeout) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - log_level debug_level = rpc_debug_level; - debug_level = rpc_debug_level; - - nssi_kill_service_args args; - - memset(&args, 0, sizeof(nssi_kill_service_args)); - args.sig = sig; - - client_init(); - - /* send the request */ - rc = nssi_call_rpc(svc, NSSI_OP_KILL_SERVICE, &args, NULL, 0, NULL, &req); - if (rc != NSSI_OK) { - log_error(debug_level, "unable to call remote method: %s", - nssi_err_str(rc)); - return rc; - } - - /* If we're forcing a kill, we don't want to wait for the response - * from the server. This will likely cause some memory leaks. */ - if (sig != 0) { - req.status = NSSI_REQUEST_COMPLETE; - } - - - /* wait for completion */ - log_debug(debug_level, "calling nssi_timedwait"); - //rc2 = nssi_timedwait(&req, timeout, &rc); - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(rpc_debug_level, "failed waiting for request %lu: %s", - req.id, - nssi_err_str(rc2)); - - /* In this case, we failed waiting for the request because - the service exited before it had a chance to send - the result. */ - /*return rc2;*/ - } - - - - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - - /* this case is a legitimate error */ - return rc; - } - - log_debug(debug_level, "Finished with nssi_kill"); - - return rc; -} - - -/** - * @brief Reset tracing on a server. - */ -int nssi_trace_reset( - const nssi_service *svc, - const char *fname, - const int ftype, - const char *enable, - const long timeout) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - static const char *empty=""; - - nssi_trace_reset_args args; - - client_init(); - - memset(&args, 0, sizeof(nssi_trace_reset_args)); - - /* initialize the args */ - args.fname = (char *)fname; - args.ftype = ftype; - args.enable = (char *)enable; - - log_debug(rpc_debug_level, "Calling nssi_trace_reset(%s, %d, %s)\n", - fname, ftype, enable); - - if (!args.fname) - args.fname = (char *)empty; - - if (!args.enable) - args.enable = (char *)empty; - - /* send the request */ - rc = nssi_call_rpc(svc, NSSI_OP_TRACE_RESET, &args, NULL, 0, NULL, &req); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "unable to call remote method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - log_debug(rpc_debug_level, "calling nssi_timedwait"); - rc2 = nssi_timedwait(&req, timeout, &rc); - if (rc2 != NSSI_OK) { - log_error(rpc_debug_level, "failed waiting for request %lu: %s", - req.id, - nssi_err_str(rc2)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; -} - -/** @brief Test a request for completion - * - * @param req the handle of the pending request. - * @return status of the operation. - */ -int nssi_test(nssi_request *req, int *rc) { - log_error(rpc_debug_level,"not supported"); - return (int)NSSI_ENOTSUP; -} - -static int setup_short_request(nssi_request *request) -{ - int rc = NSSI_OK; - int short_request_len = 0; - - trios_declare_timer(call_time); - - if (nssi_config.use_buffer_queue) { - request->short_request_hdl=trios_buffer_queue_pop(&send_bq); - log_debug(rpc_debug_level, "Popped short request buffer"); - } else { - /* allocate memory for the short request buffer */ - short_request_len = NNTI_BUFFER_SIZE(&request->svc->req_addr); - - trios_start_timer(call_time); - rc=NNTI_alloc( - &transports[request->svc->transport_id], - short_request_len, - 1, - NNTI_SEND_SRC, - request->short_request_hdl); - trios_stop_timer("NNTI_register_memory - short request", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering short request: %s", - nnti_err_str(rc)); - goto cleanup; - } - log_debug(rpc_debug_level, "Allocated short request buffer"); - } - -cleanup: - return rc; -} - -static int teardown_short_request(nssi_request *request) -{ - int rc = NSSI_OK; - - trios_declare_timer(call_time); - - if (nssi_config.use_buffer_queue) { - trios_buffer_queue_push(&send_bq, request->short_request_hdl); - } else { - trios_start_timer(call_time); - rc=NNTI_free(request->short_request_hdl); - trios_stop_timer("NNTI_unregister_memory - send req", call_time); - } - request->short_request_hdl=NULL; - - return rc; -} - -static int setup_bulk_data(nssi_request *request) -{ - int rc = NSSI_OK; - - trios_declare_timer(call_time); - - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= request->data_size)) { - log_debug(rpc_debug_level, "using buffer queue for TARGET buffer"); - request->bulk_data_hdl=trios_buffer_queue_pop(&rdma_target_bq); - assert(request->bulk_data_hdl); - NNTI_BUFFER_SIZE(request->bulk_data_hdl)=request->data_size; - /* copy the user buffer contents into RDMA buffer */ - trios_start_timer(call_time); - memcpy(NNTI_BUFFER_C_POINTER(request->bulk_data_hdl), (char *)request->data, request->data_size); - trios_stop_timer("memcpy target buf to bq", call_time); - } else { - log_debug(rpc_debug_level, "using user buffer for TARGET buffer"); - log_debug (rpc_debug_level, "Registering data buffer (size=%d)", request->data_size); - - trios_start_timer(call_time); - rc=NNTI_register_memory( - &transports[request->svc->transport_id], - (char *)request->data, - request->data_size, - 1, - (NNTI_buf_ops_t)(NNTI_GET_SRC|NNTI_PUT_DST), - request->bulk_data_hdl); - trios_stop_timer("NNTI_register_memory - data", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering data: %s", - nnti_err_str(rc)); - goto cleanup; - } - log_debug(rpc_debug_level, "Allocated bulk data buffer"); - } - - if (logging_debug(rpc_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "request->bulk_data_hdl", - "NNTI_buffer_t", request->bulk_data_hdl); - } - -cleanup: - return rc; -} - -static int teardown_bulk_data(nssi_request *request) -{ - int rc = NSSI_OK; - - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= request->data_size)) { - trios_buffer_queue_push(&rdma_target_bq, request->bulk_data_hdl); - } else { - rc=NNTI_unregister_memory(request->bulk_data_hdl); - } - request->bulk_data_hdl=NULL; - - return rc; -} - -static int setup_short_result(nssi_request *request) -{ - int rc = NSSI_OK; - - trios_declare_timer(call_time); - - if (nssi_config.use_buffer_queue) { - request->short_result_hdl=trios_buffer_queue_pop(&recv_bq); - } else { - log_debug (rpc_debug_level, "allocating short result (size=%d)", NSSI_SHORT_RESULT_SIZE); - - trios_start_timer(call_time); - rc=NNTI_alloc( - &transports[request->svc->transport_id], - NSSI_SHORT_RESULT_SIZE, - 1, - NNTI_RECV_DST, - request->short_result_hdl); - trios_stop_timer("NNTI_register_memory - short result", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering short result: %s", - nnti_err_str(rc)); - goto cleanup; - } - log_debug(rpc_debug_level, "Allocated short result buffer"); - } - -cleanup: - return rc; -} - -static int teardown_short_result(nssi_request *request) -{ - int rc = NSSI_OK; - - if (nssi_config.use_buffer_queue) { - trios_buffer_queue_push(&recv_bq, request->short_result_hdl); - } else { - rc=NNTI_free(request->short_result_hdl); - } - request->short_result_hdl=NULL; - - return rc; -} - -static int setup_long_args(nssi_request *request, nssi_size args_size) -{ - int rc = NSSI_OK; - - log_debug(rpc_debug_level,"allocating space for args"); - /* allocate memory for the encoded arguments. The request - * structure keeps track of the buffer so it can free - * the memory later. */ - rc=NNTI_alloc( - &transports[request->svc->transport_id], - args_size, - 1, - NNTI_GET_SRC, - request->long_args_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering long args: %s", - nnti_err_str(rc)); - goto cleanup; - } - -cleanup: - return rc; -} - -static int teardown_long_args(nssi_request *req) -{ - int rc = NSSI_OK; - char *buf; - - trios_declare_timer(call_time); - - if (!req->use_long_args) { - return(rc); - } - - log_debug(rpc_debug_level, "waiting for long args"); - - if (req->app_pinned_long_args == FALSE) { - rc=NNTI_free(req->long_args_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering long args: %s", - nnti_err_str(rc)); - } - } - - return rc; -} - -/** - * @brief Wait for a request to complete. - * - * @param req the handle of the pending request. - */ -int nssi_timedwait(nssi_request *req, int timeout, int *remote_rc) -{ - int rc=NSSI_OK; - log_level debug_level = rpc_debug_level; - NNTI_status_t status; - - int retries=0; - - trios_declare_timer(total_time); - trios_declare_timer(call_time); - - trios_start_timer(total_time); - - switch (req->status) { - - case NSSI_REQUEST_NULL: - - goto request_null; - - case NSSI_REQUEST_ERROR: - log_debug(rpc_debug_level,"timedwait finished"); - *remote_rc = req->error_code; - - goto request_error; - - case NSSI_REQUEST_COMPLETE: - log_debug(rpc_debug_level,"timedwait finished"); - *remote_rc = NSSI_OK; - - goto request_complete; - - default: - log_debug(debug_level, "calling NNTI_wait for request"); - /* Times out after DEFAULT_RPC_TIMEOUT */ - trios_start_timer(call_time); - rc=NNTI_wait( - &req->short_request_wr, - timeout, - &status); - trios_stop_timer("NNTI_wait - send req", call_time); - if (status.result == NNTI_EDROPPED) { - log_debug(LOG_ALL, "request dropped"); - } - while (status.result == NNTI_EDROPPED) { - if (retries++ < MAX_RETRIES) { - trios_start_timer(call_time); - rc=NNTI_send( - &req->svc->svc_host, - req->short_request_hdl, - &req->svc->req_addr, - &req->short_request_wr); - trios_stop_timer("NNTI_send - resend req", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed resending short request: %s", - nnti_err_str(rc)); - break; - } - trios_start_timer(call_time); - rc=NNTI_wait( - &req->short_request_wr, - timeout, - &status); - trios_stop_timer("NNTI_wait - resend req", call_time); - if (status.result == NNTI_EDROPPED) { - log_debug(LOG_ALL, "request dropped"); - } - } else { - log_error(rpc_debug_level, "retries exhausted: %s", - nnti_err_str(rc)); - break; - } - } - if (rc == NNTI_ETIMEDOUT) { - log_info(rpc_debug_level, "send request timed out"); - - goto wait_timedout; - } - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for send: %s", - nnti_err_str(rc)); - - goto wait_failed; - } - - /* the service is processing the request. */ - req->status = NSSI_PROCESSING_REQUEST; - - log_debug(debug_level, "calling NNTI_wait for result"); - trios_start_timer(call_time); - NNTI_create_work_request( - req->short_result_hdl, - &req->short_result_wr); - rc=NNTI_wait( - &req->short_result_wr, - timeout, - &status); - trios_stop_timer("NNTI_wait - short result", call_time); - if (status.result == NNTI_ETIMEDOUT) { - log_info(debug_level, "NNTI_wait for result timed out"); - rc = status.result; - - goto wait_timedout; - } - if (status.result != NNTI_OK) { - log_info(debug_level, "NNTI_wait for result failed"); - rc = status.result; - req->status = NSSI_REQUEST_ERROR; - - goto wait_failed; - } - - log_debug(debug_level, "Processing result status.offset=%d", status.offset); - - /* we are now ready to process the result */ - req->status = NSSI_PROCESSING_RESULT; - - rc = process_result((char *)status.start+status.offset, req); - if (rc != NSSI_OK) { - log_fatal(debug_level,"unable to process result"); - fprint_NNTI_status(logger_get_file(), "status", "FATAL", &status); - - goto wait_failed; - } - - NNTI_destroy_work_request( - &req->short_result_wr); - - goto wait_success; - } - - - - -request_null: - return rc; - -request_error: - return rc; - -request_bad_status: - /* this should only execute if something went wrong */ - log_fatal(debug_level,"invalid request status %d",req->status); - return NSSI_EBADRPC; - -request_complete: - if (req->is_responseless == TRUE) { - return rc; - } - goto buffer_cleanup; - -wait_timedout: - return rc; - -wait_failed: - /* check for an error */ - *remote_rc = req->error_code; - return rc; - -wait_success: - /* call the callback function associated with the request */ - log_debug(debug_level, "calling callback for wait(): op=%d", req->opcode); - if (req->callback) { - req->callback(req); - } - - /* check for completion */ - log_debug(debug_level,"timedwait finished"); - *remote_rc = NSSI_OK; - goto buffer_cleanup; - -buffer_cleanup: - if (req->app_pinned_short_request == FALSE) { - log_debug(debug_level, "Cleanup short request"); - rc = teardown_short_request(req); - if (rc != NSSI_OK) { - log_error(debug_level, "failed to cleanup long args"); - } - } - - if (req->app_pinned_short_result == FALSE) { - log_debug(debug_level, "Cleanup short result"); - rc = teardown_short_result(req); - if (rc != NSSI_OK) { - log_error(debug_level, "failed to cleanup long args"); - } - } - - if (req->use_long_args) { - log_debug(debug_level, "Cleanup long args"); - /* Now we need to clean up the long arguments (if they were used) */ - rc = teardown_long_args(req); - if (rc != NSSI_OK) { - log_error(debug_level, "failed to cleanup long args"); - } - } - - /* If the request has data associated with it, the data should - * be transferred by now (server would not have sent result). - * We need to unlink the MD and free the event queue. - */ - if (req->data != NULL) { - if (req->app_pinned_bulk_data == FALSE) { - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= req->data_size)) { - /* copy the RDMA buffer contents into the user buffer. - * we can't tell if the server op was get or put. - * if it was get, then this is a waste of time. - * if it was put, then this is required. - */ - trios_start_timer(call_time); - memcpy(req->data, NNTI_BUFFER_C_POINTER(req->bulk_data_hdl), req->data_size); - trios_stop_timer("memcpy bq to target buf", call_time); - } - log_debug(debug_level, "Cleanup bulk data"); - teardown_bulk_data(req); - } - } - - trios_stop_timer("nssi_timedwait - total", total_time); - - return rc; -} - -int nssi_wait(nssi_request *req, int *rc) -{ - log_debug(rpc_debug_level, "calling nssi_timedwait(timeout=-1)"); - return nssi_timedwait(req, -1, rc); -} - -/** - * @brief Wait for any request to complete. - * - * A request is not complete unless we receive the short - * result. - * - * @param req_array @input_type The array of pending requests. - * @param size @input_type The number of pending requests. - * @param timeout @input_type The time to wait for any one request. - * - */ -int nssi_waitany( - nssi_request *req_array, - nssi_size req_count, - int timeout, - int *which, - int *remote_rc) -{ - NNTI_result_t nnti_rc=NNTI_OK; /* result of NNTI functions */ - - int rc = NSSI_OK; /* return code */ - int rc2; - int i; - - int errors =0; - int timeouts=0; - - NNTI_work_request_t **work_requests=new NNTI_work_request_t*[req_count]; - uint32_t nnti_which=0; - NNTI_status_t status; - - log_level debug_level = rpc_debug_level; - - int retries=0; - - trios_declare_timer(call_time); - - - for (i=0;isvc_host, - req_array[*which].short_request_hdl, - &req_array[*which].svc->req_addr, - &req_array[*which].short_request_wr); - trios_stop_timer("NNTI_send - resend req", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed resending short request: %s", - nnti_err_str(rc)); - req_array[*which].status = NSSI_REQUEST_ERROR; - } else { - goto wait_again; - } - } else { - log_error(rpc_debug_level, "retries exhausted: %s", - nnti_err_str(rc)); - } - } - if (status.result == NNTI_ETIMEDOUT) { - log_info(debug_level, "NNTI_wait for result timed out"); - rc = status.result; - return rc; - } - if (status.result != NNTI_OK) { - log_info(debug_level, "NNTI_wait for result failed"); - rc = status.result; - req_array[*which].status = NSSI_REQUEST_ERROR; - work_requests[*which]=NULL; - } - - if (req_array[*which].status == NSSI_SENDING_REQUEST) { - /* the send is complete. now wait for the result. */ - req_array[*which].status = NSSI_PROCESSING_REQUEST; - - nnti_rc = NNTI_create_work_request( - req_array[*which].short_result_hdl, - &req_array[*which].short_result_wr); - work_requests[*which]=&req_array[*which].short_result_wr; - - goto wait_again; - } - - log_debug(debug_level, "Processing result status.offset=%d", status.offset); - - /* we are now ready to process the result */ - req_array[*which].status = NSSI_PROCESSING_RESULT; - rc = process_result((char *)status.start+status.offset, &req_array[*which]); - if (rc != NSSI_OK) { - log_fatal(debug_level,"unable to process result"); - fprint_NNTI_status(logger_get_file(), "status", "FATAL", &status); - return rc; - } - - if (req_array[*which].use_long_args) { - log_debug(debug_level, "Cleanup long args"); - /* Now we need to clean up the long arguments (if they were used) */ - rc = teardown_long_args(&req_array[*which]); - if (rc != NSSI_OK) { - log_error(debug_level, "failed to cleanup long args"); - return NSSI_EBADRPC; - } - } - -cleanup: - - for (i=0;i= req_array[*which].data_size)) { - /* copy the RDMA buffer contents into the user buffer. - * we can't tell if the server op was get or put. - * if it was get, then this is a waste of time. - * if it was put, then this is required. - */ - trios_start_timer(call_time); - memcpy(req_array[*which].data, NNTI_BUFFER_C_POINTER(req_array[*which].bulk_data_hdl), req_array[*which].data_size); - trios_stop_timer("memcpy bq to target buf", call_time); - } - teardown_bulk_data(&req_array[*which]); - } - } - - - /* check for an error */ - if (req_array[*which].status == NSSI_REQUEST_ERROR) { - *remote_rc = req_array[*which].error_code; - return rc; - } - - /* check for completion */ - if (req_array[*which].status == NSSI_REQUEST_COMPLETE) { - log_debug(debug_level,"timedwait finished"); - *remote_rc = NSSI_OK; - return rc; - } - - /* this should only execute if something went wrong */ - log_fatal(debug_level,"invalid request status %d",req_array[*which].status); - return NSSI_EBADRPC; - -} - -/** - * @brief Wait for all requests to complete. - * - * A request is not complete unless we receive the short - * result. - * - * @param req_array @input_type The array of pending requests. - * @param size @input_type The number of pending requests. - * @param timeout @input_type The time to wait for any one request. - * - */ -int nssi_waitall( - nssi_request *req_array, - nssi_size req_count, - int timeout) -{ - NNTI_result_t nnti_rc=NNTI_OK; /* result of NNTI functions */ - - int rc = NSSI_OK; /* return code */ - int rc2; - int i; - - int errors =0; - int timeouts=0; - - NNTI_work_request_t **work_requests=new NNTI_work_request_t*[req_count]; - NNTI_status_t status; - - uint32_t which=0; - - log_level debug_level = rpc_debug_level; - - int retries=0; - - trios_declare_timer(call_time); - - - for (i=0;isvc_host, - req_array[which].short_request_hdl, - &req_array[which].svc->req_addr, - &req_array[which].short_request_wr); - trios_stop_timer("NNTI_send - resend req", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed resending short request: %s", - nnti_err_str(rc)); - break; - } - trios_start_timer(call_time); - goto wait_again; - } else { - log_error(rpc_debug_level, "retries exhausted: %s", - nnti_err_str(rc)); - } - } - if (status.result == NNTI_ETIMEDOUT) { - log_info(debug_level, "work_request[%d] timed out", which); - req_array[which].status = NSSI_REQUEST_TIMEDOUT; - timeouts++; - continue; - } - if (status.result != NNTI_OK) { - log_info(debug_level, "work_request[%d] failed", which); - req_array[which].status = NSSI_REQUEST_ERROR; - errors++; - goto wr_error; - } - - if (req_array[which].status == NSSI_SENDING_REQUEST) { - /* the send is complete. now wait for the result. */ - req_array[which].status = NSSI_PROCESSING_REQUEST; - - nnti_rc = NNTI_create_work_request( - req_array[which].short_result_hdl, - &req_array[which].short_result_wr); - work_requests[which]=&req_array[which].short_result_wr; - - goto wait_again; - } - - log_debug(debug_level, "Processing result status.offset=%d", status.offset); - - /* we are now ready to process the result */ - req_array[which].status = NSSI_PROCESSING_RESULT; - rc = process_result((char *)status.start+status.offset, &req_array[which]); - if (rc != NSSI_OK) { - log_fatal(debug_level,"unable to process result"); - fprint_NNTI_status(logger_get_file(), "status", "FATAL", &status); - req_array[which].status = NSSI_REQUEST_ERROR; - errors++; - } - - if (req_array[which].use_long_args) { - log_debug(debug_level, "Cleanup long args"); - /* Now we need to clean up the long arguments (if they were used) */ - rc = teardown_long_args(&req_array[which]); - if (rc != NSSI_OK) { - log_error(debug_level, "failed to cleanup long args"); - req_array[which].status = NSSI_REQUEST_ERROR; - errors++; - } - } - - NNTI_destroy_work_request(work_requests[which]); - - /* call the callback function associated with the request */ - log_debug(debug_level, "calling req_array[%d] callback: op=%d", which, req_array[which].opcode); - if (req_array[which].callback) { - req_array[which].callback(&req_array[which]); - } - - if (req_array[which].app_pinned_short_result == FALSE) { - teardown_short_result(&req_array[which]); - } - - /* If the request has data associated with it, the data should - * be transferred by now (server would not have sent result). - * We need to unlink the MD and free the event queue. - */ - if (req_array[which].data != NULL) { - if (req_array[which].app_pinned_bulk_data == FALSE) { - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= req_array[which].data_size)) { - /* copy the RDMA buffer contents into the user buffer. - * we can't tell if the server op was get or put. - * if it was get, then this is a waste of time. - * if it was put, then this is required. - */ - trios_start_timer(call_time); - memcpy(req_array[which].data, NNTI_BUFFER_C_POINTER(req_array[which].bulk_data_hdl), req_array[which].data_size); - trios_stop_timer("memcpy bq to target buf", call_time); - } - teardown_bulk_data(&req_array[which]); - } - } - -wr_error: - work_requests[which] = NULL; - } - - delete(work_requests); - - if (errors > 0) { - return NSSI_EBADRPC; - } - - if (timeouts > 0) { - return NSSI_ETIMEDOUT; - } - - return rc; -} - -/** - * @brief Initialize portals data structures associated with an - * RPC request. - * - * @ingroup rpc_ptl_impl - * - * This method checks to see if the operation arguments for a - * request arguments fit into the short request buffer (along - * with the request header). If not, this method creates the necessary - * portals data structures on the client that enable the server to - * fetch the arguments explicitely. - * - */ -static int encode_args( - const nssi_service *svc, - void *args, - NNTI_buffer_t *short_req_hdl, - nssi_request_header *header, - nssi_request *request) -{ - - int rc = NSSI_OK; /* return code */ - - trios_declare_timer(call_time); - - /* xdrs for the header and the args. */ - XDR hdr_xdrs, args_xdrs; - - nssi_size hdr_size = 0; - nssi_size args_size = 0; - nssi_size data_size = 0; - nssi_size data_offset = 0; - nssi_size remaining = 0; - - char *buf=NULL; - - - if (logging_debug(rpc_debug_level)) { - fprint_nssi_service(logger_get_file(), "service", "DEBUG", svc); - } - /* get the encoding functions for the request */ - switch (svc->rpc_encode) { - case NSSI_RPC_XDR: - rc = nssi_lookup_xdr_encoding(request->opcode, - &request->xdr_encode_args, - &request->xdr_encode_data, - &request->xdr_decode_result); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, - "could not find xdr encoding functions"); - goto cleanup; - } - break; - - default: - log_error(rpc_debug_level, "invalid encoding scheme"); - rc = NSSI_EENCODE; - goto cleanup; - } - - /* set the request ID for the header */ - header->id = request->id; - - - /* the size of the encoded header */ - hdr_size = xdr_sizeof((xdrproc_t)&xdr_nssi_request_header, header); - - if (args == NULL) { - /* there are no args */ - header->fetch_args = FALSE; - args_size = 0; - } else { - /* the size of the encoded args */ - args_size = xdr_sizeof(request->xdr_encode_args, args); - /* what's left after encoding the header */ - remaining = NNTI_BUFFER_SIZE(short_req_hdl) - hdr_size; - if (remaining > args_size) { - /* args will fit in the request. fetch not required. */ - header->fetch_args = FALSE; - } else { - /* args will not fit in the request. server must fetch. */ - header->fetch_args = TRUE; - - if (request->is_responseless == TRUE) { - log_debug(rpc_debug_level, "illegal combination: responseless + long args"); - return NSSI_EINVAL; - } - } - } - - log_debug(rpc_debug_level, "initial hdr_size=%d", hdr_size); - - /** - * If the arguments are too large to fit in an - * \ref short request buffer, the client stores - * excess arguments and waits for the server to - * "fetch" the arguments using the \ref nssi_get_data method. - */ - if (header->fetch_args == TRUE) { - - log_debug(rpc_debug_level,"putting args (len=%d) " - "in long request", args_size); - - request->use_long_args=1; - - /* the app may provide its own registered long args buffer. check here. */ - if (request->app_pinned_long_args == FALSE) { - setup_long_args(request, args_size); - } - header->args_addr=*request->long_args_hdl; - - /* create an xdr memory stream for the encoded args */ - xdrmem_create(&args_xdrs, NNTI_BUFFER_C_POINTER(request->long_args_hdl), - NNTI_BUFFER_SIZE(request->long_args_hdl), XDR_ENCODE); - - /* if we get here, args should not be NULL */ - assert(args); - - /* encode the args */ - log_debug(rpc_debug_level,"encoding args"); - trios_start_timer(call_time); - if (! request->xdr_encode_args(&args_xdrs, args)) { - log_fatal(rpc_debug_level,"failed to encode the args"); - return NSSI_EENCODE; - } - trios_stop_timer("xdr_encode_args - encode", call_time); - - } else if ((header->fetch_args == FALSE) && (args != NULL)) { - - log_debug(rpc_debug_level,"putting args (len=%d) " - "in short request", args_size); - - request->use_long_args=0; - - /* create an xdr memory stream inside the short request for the encoded args */ - xdrmem_create(&args_xdrs, NNTI_BUFFER_C_POINTER(short_req_hdl) + hdr_size, - NNTI_BUFFER_SIZE(short_req_hdl) - hdr_size, XDR_ENCODE); - - /* if we get here, args should not be NULL */ - assert(args); - - /* encode the args */ - log_debug(rpc_debug_level,"encoding args"); - trios_start_timer(call_time); - if (! request->xdr_encode_args(&args_xdrs, args)) { - log_fatal(rpc_debug_level,"failed to encode the args"); - return NSSI_EENCODE; - } - trios_stop_timer("xdr_encode_args - encode", call_time); - - } - - /* if the args are being fetched, then the header has changed. - * recalculate the size of the encoded header. */ - if (header->fetch_args == TRUE) { - hdr_size = xdr_sizeof((xdrproc_t)&xdr_nssi_request_header, header); - log_debug(rpc_debug_level, "using long args. recalculated hdr_size=%d", hdr_size); - } - - if (nssi_config.put_data_in_request == FALSE) { - header->fetch_data = TRUE; - } else { - /* the size of the bulk data buffer */ - data_size=NNTI_BUFFER_SIZE(&header->data_addr); - if (data_size > 0) { - /* what's left after encoding the header into the short request */ - remaining = NNTI_BUFFER_SIZE(short_req_hdl) - hdr_size; - data_offset = hdr_size; - if (header->fetch_args == FALSE) { - /* the args are in the short request. reduce the remaining space by the size of the encoded args. */ - remaining -= args_size; - data_offset += args_size; - } - if (remaining > data_size) { - /* data will fit in the request. fetch not required. */ - header->fetch_data = FALSE; - } else { - /* data will not fit in the request. server must fetch. */ - header->fetch_data = TRUE; - } - } else { - /* there is no data */ - header->fetch_data = FALSE; - } - - if ((data_size > 0) && (header->fetch_data == FALSE)) { - log_debug(rpc_debug_level,"putting data (size=%d, offset=%d) in short request", data_size, data_offset); - /* copy small data into the short request */ - memcpy(NNTI_BUFFER_C_POINTER(short_req_hdl)+data_offset, NNTI_BUFFER_C_POINTER(&header->data_addr), data_size); - } - } - - - /* - * encode the header - */ - log_debug(rpc_debug_level,"encoding request header"); - - /* create an xdr memory stream for the short request buffer */ - xdrmem_create(&hdr_xdrs, NNTI_BUFFER_C_POINTER(short_req_hdl), - NNTI_BUFFER_SIZE(short_req_hdl), XDR_ENCODE); - - trios_start_timer(call_time); - if (! xdr_nssi_request_header(&hdr_xdrs, header)) { - log_fatal(rpc_debug_level,"failed to encode the request header"); - return NSSI_EENCODE; - } - trios_stop_timer("xdr_nssi_request_header - encode", call_time); - - - log_debug(rpc_debug_level, "short_req_size=%d, " - "hdr_size(final)=%d, args_size=%d, data_size=%d", - NNTI_BUFFER_SIZE(short_req_hdl), hdr_size, args_size, data_size); - - /* print the header for debugging */ - if (logging_debug(rpc_debug_level)) { - fprint_nssi_request_header(logger_get_file(), "header", - "nssi_request_header", header); - } - -cleanup: - - /* done! */ - return rc; -} - - -/** - * @brief Setup an RPC request. - * - * @ingroup rpc_ptl_impl - * - * This method populates an RPC request. All input parameters are copied - * into the request. The request can be used to send the RPC to the service - * and wait for the result - * - * @param svc @input descriptor for the NSSI service. - * @param opcode @input descriptor for the remote method. - * @param args @input pointer to the arguments. - * @param data @input pointer to data (for bulk data transfers). - * @param data_size @input length of data buffer - * @param result @input where to put results. - * @param request @output The request handle (used to test for - * completion). - */ -int nssi_create_request( - const nssi_service *svc, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *result, - nssi_request *request) -{ - /* local count does not need protection */ - static int64_t local_count; - - /* local variables */ - int rc=NSSI_OK; /* return code */ - - log_level debug_level = rpc_debug_level; - - char *buf; - - int retries=0; - - trios_declare_timer(total_time); - - unsigned long len=0; - - trios_start_timer(total_time); - - log_debug(rpc_debug_level, "enter"); - - /* increment global counter */ - local_count = nthread_counter_increment(&request_count); - if (local_count == -1) { - log_error(debug_level, "Unable to increment counter"); - goto cleanup; - } - - /*------ Initialize variables and buffers ------*/ - memset(request, 0, sizeof(nssi_request)); - - /* set request fields */ - request->svc = svc; - request->id = local_count; /* id of the request (used for debugging) */ - request->opcode = opcode; /* operation ID */ - request->is_responseless = FALSE; - request->is_sync = FALSE; - request->sync_timeout = -1; - request->error_code = NSSI_OK; /* return code of remote method */ - request->status = NSSI_SENDING_REQUEST; /* status of this request */ - request->app_pinned_short_request = FALSE; - request->short_request_hdl = &request->short_request; - request->app_pinned_long_args = FALSE; - request->long_args_hdl = &request->long_args; - request->args = args; /* where to find the RPC args */ - request->app_pinned_bulk_data = FALSE; - request->bulk_data_hdl = &request->bulk_data; - request->data_size = data_size; - request->data = (data_size > 0) ? data : NULL; - request->app_pinned_short_result = FALSE; - request->short_result_hdl = &request->short_result; - request->result = result; /* where to put the result */ - -cleanup: - trios_stop_timer("nssi_create_request - total", total_time); - log_debug(rpc_debug_level, "exit"); - - return rc; -} - -/** - * @brief Send an RPC request to an NSSI server. - * - * @ingroup rpc_ptl_impl - * - * This method encodes and transfers an RPC request header and - * operation arguments to an NSSI server using NNTI. If the - * arguments are sufficiently small, \b nssi_send_request sends - * the request header and the arguments in a single message. - * If the arguments are large (i.e., too large for the request buffer), - * the server fetches the arguments from a client-side buffer. - * - * @param request @inout The request handle (used to test for completion). - */ -int nssi_send_request( - nssi_request *request) -{ - /* local variables */ - int rc=NSSI_OK; /* return code */ - - NNTI_status_t status; - - log_level debug_level = rpc_debug_level; - - nssi_request_header header; /* the request header */ - char *buf; - - trios_declare_timer(total_time); - trios_declare_timer(call_time); - - unsigned long len=0; - - trios_start_timer(total_time); - - log_debug(rpc_debug_level, "enter"); - - if (request->is_responseless == TRUE) { - /* quick check for illegal request setup */ - if (request->app_pinned_short_result == TRUE) { - log_debug(rpc_debug_level, "illegal combination: responseless + app pinned short result"); - return NSSI_EINVAL; - } - if ((request->app_pinned_short_result == FALSE) && (request->result)) { - log_debug(rpc_debug_level, "illegal combination: responseless + short result NOT NULL"); - return NSSI_EINVAL; - } - if (request->data_size > 0) { - log_debug(rpc_debug_level, "illegal combination: responseless + bulk data"); - return NSSI_EINVAL; - } - } - - - /*------ Initialize variables and buffers ------*/ - memset(&header, 0, sizeof(nssi_request_header)); - - /* set the opcode for the request header */ - header.opcode = request->opcode; - - /* the app may provide its own registered request buffer. check here. */ - if (request->app_pinned_short_request == FALSE) { - /* the app didn't provide a registered request buffer. get one here. */ - setup_short_request(request); - } - - if (request->data_size > 0) { - - /* the app may provide its own registered bulk data buffer. check here. */ - if (request->app_pinned_bulk_data == FALSE) { - /* the app didn't provide a registered bulk data buffer. get one here. */ - setup_bulk_data(request); - } - header.data_addr=*request->bulk_data_hdl; - } - - header.is_responseless = request->is_responseless; - if (request->is_responseless == FALSE) { - /* the app may provide its own registered result buffer. check here. */ - if (request->app_pinned_short_result == FALSE) { - /* the app didn't provide a registered result buffer. get one here. */ - setup_short_result(request); - } - header.res_addr=*request->short_result_hdl; - } - - /* --- encode the arguments (might place args in the short request) --- */ - trios_start_timer(call_time); - rc = encode_args(request->svc, request->args, request->short_request_hdl, &header, request); - trios_stop_timer("encode_args", call_time); - if (rc != NSSI_OK) { - log_fatal(rpc_debug_level, "unable to encode arguments"); - goto cleanup; - } - - log_debug(LOG_OFF, "header.id=%d", header.id); - - /* get the number of valid bytes in the request */ - trios_start_timer(call_time); - len = xdr_sizeof((xdrproc_t)&xdr_nssi_request_header, &header); - trios_stop_timer("xdr_sizeof - request header", call_time); - - /* if the args in the short request, add args len */ - if (!header.fetch_args) { - len += NNTI_BUFFER_SIZE(&header.args_addr); - } - - /* send the encoded short request buffer to the server */ - log_debug(rpc_debug_level,"sending short request, id=%lu, len=%d", header.id, len); - - trios_start_timer(call_time); - rc=NNTI_send( - &request->svc->svc_host, - request->short_request_hdl, - &request->svc->req_addr, - &request->short_request_wr); - trios_stop_timer("NNTI_send - send req", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed sending short request: %s", - nnti_err_str(rc)); - goto cleanup; - } - log_debug(rpc_debug_level,"message sent"); - - if (request->is_responseless == TRUE) { - trios_start_timer(call_time); - rc=NNTI_wait( - &request->short_request_wr, - -1, - &status); - trios_stop_timer("NNTI_wait - send req", call_time); - if (status.result == NNTI_EDROPPED) { - log_debug(rpc_debug_level, "request dropped"); - } - if (rc != NNTI_OK) { - log_debug(rpc_debug_level, "failed waiting for send: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - - if (request->is_sync == TRUE) { - int remote_rc=NSSI_OK; - rc=nssi_timedwait(request, request->sync_timeout, &remote_rc); - } - -cleanup: - log_debug(rpc_debug_level, "finished nssi_send_request (req.opcode=%d, req.id=%d)", - request->opcode, request->id); - - if (rc == NSSI_OK) { - if (request->is_responseless == TRUE) { - request->status = NSSI_REQUEST_COMPLETE; - } - } else { - /* change the state of the pending request */ - request->status = NSSI_REQUEST_ERROR; - - if (request->app_pinned_short_request == FALSE) { - teardown_short_request(request); - } - - if (request->app_pinned_bulk_data == FALSE) { - if (request->data_size > 0) { - teardown_bulk_data(request); - } - } - if (request->app_pinned_short_result == FALSE) { - teardown_short_result(request); - } - } - - trios_stop_timer("nssi_send_request - total", total_time); - - return rc; -} - - -/** - * @brief Setup and send an RPC request to an NSSI server. - * - * @ingroup rpc_ptl_impl - * - * This method populates, encodes and transfers an RPC request header and - * operation arguments to an NSSI server using NNTI. If the - * arguments are sufficiently small, \b nssi_call_rpc sends - * the request header and the arguments in a single message. - * If the arguments are large (i.e., too large for the request buffer), - * the server fetches the arguments from a client-side buffer. - * - * @param svc @input descriptor for the NSSI service. - * @param opcode @input descriptor for the remote method. - * @param args @input pointer to the arguments. - * @param data @input pointer to data (for bulk data transfers). - * @param data_size @input length of data buffer - * @param result @input where to put results. - * @param request @output The request handle (used to test for - * completion). - */ -int nssi_call_rpc( - const nssi_service *svc, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *result, - nssi_request *request) -{ - /* local variables */ - int rc=NSSI_OK; /* return code */ - - log_level debug_level = rpc_debug_level; - - nssi_request_header header; /* the request header */ - - int retries=0; - - trios_declare_timer(total_time); - trios_declare_timer(call_time); - - unsigned long len=0; - - trios_start_timer(total_time); - - log_debug(rpc_debug_level, "enter"); - - rc=nssi_create_request(svc, opcode, args, data, data_size, result, request); - if (rc != NSSI_OK) { - goto cleanup; - } - rc=nssi_send_request(request); - if (rc != NSSI_OK) { - goto cleanup; - } - -cleanup: - - return rc; - -} - - -/** - * @brief Send an RPC request to an NSSI server. - * - * @ingroup rpc_ptl_impl - * - * This method encodes and transfers an RPC request header and - * operation arguments to an NSSI server using Portals. If the - * arguments are sufficiently small, \b nssi_call_rpc sends - * the request header and the arguments in a single message. - * If the arguments are large (i.e., too large for the request buffer), - * the server to fetch the arguments from a client-side portal. - * - * @param rpc @input descriptor for the remote method. - * @param args @input pointer to the arguments. - * @param data @input pointer to data (for bulk data transfers). - * @param data_size @input length of data buffer - * @param result @input where to put results. - * @param req @output The request handle (used to test for - * completion). - */ - -int nssi_call_rpc_sync( - const nssi_service *svc, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *result) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - rc = nssi_call_rpc(svc, opcode, args, data, data_size, result, &req); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "failed calling nssi_call_rpc: %s", nssi_err_str(rc)); - goto cleanup; - } - - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(rpc_debug_level, "failed waiting for result from server: %s", nssi_err_str(rc)); - rc = rc2; - goto cleanup; - } - - if (rc != NSSI_OK) { - log_warn(rpc_debug_level, "error in remote method: %s", - nssi_err_str(rc)); - return rc; - } - -cleanup: - return rc; -} - -int nssi_multicast_rpc( - const nssi_service *svcs, - const nssi_size num_svcs, - const int opcode, - void *args, - void *data, - uint32_t data_size, - void *results, - nssi_size result_size, /* size in bytes of the result */ - nssi_request *requests) -{ - nssi_size i=0; - int rc=0; - - for (i=0;i reqs(num_svcs); - - rc = nssi_multicast_rpc(svcs, num_svcs, opcode, args, data, data_size, results, result_size, &reqs[0]); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "failed calling nssi_call_rpc: %s", nssi_err_str(rc)); - return rc; - } - - rc2 = nssi_waitall(&reqs[0], num_svcs, -1); - if (rc2 != NSSI_OK) { - log_error(rpc_debug_level, "failed waiting for result from server: %s", nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_warn(rpc_debug_level, "error in remote method: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; -} - -int nssi_atomic_increment( - const nssi_service *svc, - const uint64_t remote_atomic, - const uint64_t local_atomic) -{ - int rc=NSSI_OK; /* return code */ - - NNTI_work_request_t wr; - NNTI_status_t status; - - rc=NNTI_atomic_fop(&transports[svc->transport_id], - &svc->svc_host, - remote_atomic, - local_atomic, - 1, - NNTI_ATOMIC_FADD, - &wr); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - rc=NNTI_wait(&wr, -1, &status); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - - return(rc); -} - -int nssi_atomic_decrement( - const nssi_service *svc, - const uint64_t remote_atomic, - const uint64_t local_atomic) -{ - int rc=NSSI_OK; /* return code */ - - NNTI_work_request_t wr; - NNTI_status_t status; - - rc=NNTI_atomic_fop(&transports[svc->transport_id], - &svc->svc_host, - remote_atomic, - local_atomic, - -1, - NNTI_ATOMIC_FADD, - &wr); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - rc=NNTI_wait(&wr, -1, &status); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - - return(rc); -} - -int nssi_atomic_read( - const nssi_service *svc, - const uint64_t local_atomic, - int64_t *value) -{ - int rc=NSSI_OK; /* return code */ - - NNTI_work_request_t wr; - NNTI_status_t status; - - rc=NNTI_atomic_read( - &transports[svc->transport_id], - local_atomic, - value); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "atomic read failed: %s", - nnti_err_str(rc)); - return rc; - } - - return(rc); -} diff --git a/packages/trios/libraries/nessie/nnti/CMakeLists.txt b/packages/trios/libraries/nessie/nnti/CMakeLists.txt deleted file mode 100644 index 036cb3ca425f..000000000000 --- a/packages/trios/libraries/nessie/nnti/CMakeLists.txt +++ /dev/null @@ -1,130 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -SET(NNTI_HEADERS "") -SET(NNTI_NONINSTALLHEADERS "") -SET(NNTI_SOURCES "") - - -############# COMMON ########################## - -TRIBITS_SUBPACKAGE(nnti) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -############# NNTI ########################## - -SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_POSIX_C_SOURCE=200112L") - -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/Trios_nnti_xdr.x) - -APPEND_SET(NNTI_NOINSTHEADERS - nnti_gni.h - nnti_ib.h - nnti_dcmf.h - nnti_mpi.h - nnti_internal.h - nnti_ptls.h - nnti_utils.h - buffer_queue.h -) - -APPEND_SET(NNTI_HEADERS - ${CMAKE_CURRENT_BINARY_DIR}/Trios_nnti_xdr.h - Trios_nnti_fprint_types.h - Trios_nnti.h -) - -APPEND_SET(NNTI_SOURCES - ${CMAKE_CURRENT_BINARY_DIR}/Trios_nnti_xdr.c - fprint_types.cpp - nnti.c - nnti_utils.c - buffer_queue.cpp -) - -SET(TRIOS_SUPPORTED_NETWORK_FOUND 0) -IF (${PACKAGE_NAME}_ENABLE_Portals OR ${PACKAGE_NAME}_ENABLE_CrayPortals) - APPEND_SET(NNTI_SOURCES nnti_ptls.cpp) - SET(TRIOS_SUPPORTED_NETWORK_FOUND 1) -ENDIF () -IF (${PACKAGE_NAME}_ENABLE_InfiniBand) - APPEND_SET(NNTI_SOURCES nnti_ib.cpp) - SET(TRIOS_SUPPORTED_NETWORK_FOUND 1) -ENDIF () -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(NNTI_SOURCES nnti_gni.cpp) - SET(TRIOS_SUPPORTED_NETWORK_FOUND 1) -ENDIF () -IF (${PARENT_PACKAGE_NAME}_ENABLE_BGPDCMF) - APPEND_SET(NNTI_SOURCES nnti_dcmf.cpp) - SET(TRIOS_SUPPORTED_NETWORK_FOUND 1) -ENDIF () -IF (${PARENT_PACKAGE_NAME}_ENABLE_BGQPAMI) - APPEND_SET(NNTI_SOURCES nnti_pami.cpp) - SET(TRIOS_SUPPORTED_NETWORK_FOUND 1) -ENDIF () -IF (${PARENT_PACKAGE_NAME}_ENABLE_MPI) - APPEND_SET(NNTI_SOURCES nnti_mpi.cpp) - SET(TRIOS_SUPPORTED_NETWORK_FOUND 1) -ENDIF () -IF (NOT TRIOS_SUPPORTED_NETWORK_FOUND) - message(FATAL "Did not find a supported network protocol. ") -ENDIF () - -############# libtrios_nnti.a ########################## - -TRIBITS_ADD_LIBRARY( - trios_nnti - HEADERS ${NNTI_HEADERS} - NOINSTALLHEADERS ${NNTI_NOINSTALLHEADERS} - SOURCES ${NNTI_SOURCES} - DEPLIBS ${DEPLIBS} -) - - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/libraries/nessie/nnti/Trios_nnti.h b/packages/trios/libraries/nessie/nnti/Trios_nnti.h deleted file mode 100644 index fe1cc362f650..000000000000 --- a/packages/trios/libraries/nessie/nnti/Trios_nnti.h +++ /dev/null @@ -1,597 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file Trios_nnti.h - * - * @brief Trios_nnti.h - * - * @author Todd Kordenbrock (thkorde\@sandia.gov). - * Created on: Jan 13, 2011 - */ - -#ifndef NNTI_H_ -#define NNTI_H_ - - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** @addtogroup nnti_impl - * @{ - */ - -typedef NNTI_result_t (*NNTI_callback_fn_t) ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - void *context); - - -/** - * @brief Initialize NNTI to use a specific transport. - * - * \param[in] trans_id The ID of the transport the client wants to use. - * \param[in] my_url A string that describes the transport parameters. - * \param[out] trans_hdl A handle to the configured transport. - * \return A result code (NNTI_OK or an error) - * - */ -NNTI_result_t NNTI_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -/** - * @brief Return the URL field of this transport. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[out] url A string that describes this process in a transport specific way. - * \param[in] maxlen The length of the 'url' string parameter. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -/** - * @brief Prepare for communication with the peer identified by url. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] url A string that describes a peer in a transport specific way. - * \param[in] timeout The amount of time (in milliseconds) to wait before giving up. - * \param[out] peer_hdl A handle to a peer that can be used for network operations. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -/** - * @brief Terminate communication with this peer. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] peer_hdl A handle to a peer. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -/** - * @brief Allocate a block of memory and prepare it for network operations. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] size The size (in bytes) of 'buffer'. - * \param[in] ops Allowable transport ops for 'buffer'. - * \param[out] reg_buf A pointer to a NNTI_buffer_t. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -/** - * @brief Cleanup after network operations are complete and release the block - * of memory. - * - * \param[in] reg_buf The buffer to cleanup. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_free ( - NNTI_buffer_t *reg_buf); - -/** - * @brief Prepare a block of memory for network operations. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] buffer Pointer to a memory block. - * \param[in] size The size (in bytes) of 'buffer'. - * \param[in] ops Allowable transport ops for 'buffer'. - * \param[out] reg_buf A pointer to a NNTI_buffer_t. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -/** - * @brief Prepare a list of memory segments for network operations. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] segments List of memory segments. - * \param[in] segment_lengths List of segment lengths (in bytes). - * \param[in] num_segments The number of segments in the list. - * \param[in] ops Allowable transport ops. - * \param[out] reg_buf A pointer to a NNTI_buffer_t. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -/** - * @brief Cleanup after network operations are complete. - * - * \param[in] reg_buf The buffer to cleanup. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_unregister_memory ( - NNTI_buffer_t *reg_buf); - -/** - * @brief Calculate the number of bytes required to store an encoded NNTI data structure. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] nnti_dt The NNTI data structure cast to void*. - * \param[out] packed_len The number of bytes required to store the encoded data structure. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_dt_sizeof ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt, - uint64_t *packed_len); - -/** - * @brief Encode an NNTI data structure into an array of bytes. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] nnti_dt The NNTI data structure cast to void*. - * \param[in] packed_buf A array of bytes to store the encoded data structure. - * \param[in] packed_buflen The length of packed_buf. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_dt_pack ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt, - char *packed_buf, - const uint64_t packed_buflen); - -/** - * @brief Decode an array of bytes into an NNTI datatype. - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] nnti_dt The NNTI data structure cast to void*. - * \param[in] packed_buf A array of bytes containing the encoded data structure. - * \param[out] packed_len The number of encoded bytes. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_dt_unpack ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt, - char *packed_buf, - uint64_t packed_buflen); - -/** - * @brief Free a variable size NNTI datatype that was unpacked with NNTI_dt_unpack(). - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] nnti_dt The NNTI data structure cast to void*. - */ -NNTI_result_t NNTI_dt_free ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt); - -/** - * @brief Send a message to a peer. - * - * \param[in] peer_hdl The peer to send the message to. - * \param[in] msg_hdl A buffer containing the message to send. - * \param[in] dest_hdl A buffer to put the data into. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr); - -/** - * @brief Transfer data to a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to put. - * \param[in] src_offset The offset (in bytes) into the src_buffer from which to put. - * \param[in] src_length The number of bytes to put. - * \param[in] dest_buffer_hdl A buffer to put the data into. - * \param[in] dest_offset The offset (in bytes) into the dest at which to put. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -/** - * @brief Transfer data from a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to get. - * \param[in] src_offset The offset (in bytes) into the src_buffer from which to get. - * \param[in] src_length The number of bytes to get. - * \param[in] dest_buffer_hdl A buffer to get the data into. - * \param[in] dest_offset The offset (in bytes) into the dest at which to get. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -/** - * @brief Transfer data to a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to put. - * \param[in] src_length The number of bytes to put. - * \param[in] dest_buffer_list A list of buffers to put the data into. - * \param[in] dest_count The number of destination buffers. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr); - - -/** - * @brief Transfer data from a peer. - * - * \param[in] src_buffer_list A list of buffers containing the data to get. - * \param[in] src_length The number of bytes to get. - * \param[in] src_count The number of source buffers. - * \param[in] dest_buffer_hdl A buffer to get the data into. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr); - - -/** - * assign a callback to an atomic variable - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] local_atomic index of the local result variable - * \param[in] cbfunc callback function invoked when the atomic variable is modified - * \param[in] context data passed to cbfunc() at every invocation - * - * This function assigns a callback to the atomic variable {local_atomic}. - * {cbfunc} will be invoked after every successful atomic operation on - * {local_atomic}. Reads are not considered an atomic operation. - */ -NNTI_result_t NNTI_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context); - -/** - * read a 64-bit value from an local atomic variable - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] local_atomic index of the local atomic variable - * \param[out] value current value of the atomic variable - * - * - */ -NNTI_result_t NNTI_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value); - - -/** - * perform a 64-bit atomic operation with get semantics - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] peer NNTI process that hosts the target atomic variable - * \param[in] target_atomic index of the target atomic variable - * \param[in] result_atomic index of the local result atomic variable - * \param[in] operand 64-bit operand to the atomic operation - * \param[in] op atomic operation to execute - * \param[out] wr work request to wait on - * - * This function executes an atomic operation with get semantics. When the operation - * is complete the result of the operation is visible in {target_atomic} and the - * previous value is visible in {result_atomic}. - */ -NNTI_result_t NNTI_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr); - -/** - * perform a 64-bit compare-and-swap operation - * - * \param[in] trans_hdl A handle to the configured transport. - * \param[in] peer NNTI process that hosts the target atomic variable - * \param[in] target_atomic index of the target atomic variable - * \param[in] result_atomic index of the local result atomic variable - * \param[in] compare_operand 64-bit operand to compare with - * \param[in] swap_operand 64-bit operand to swap in - * \param[out] wr work request to wait on - * - * This function executes an atomic operation with get semantics. When the operation - * is complete the result of the operation is visible in (target_atomic} and the - * previous value is visible in {result_atomic}. - */ -NNTI_result_t NNTI_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr); - - -/** - * @brief Create a receive work request that can be used to wait for buffer - * operations to complete. - * - */ -NNTI_result_t NNTI_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr); - - -/** - * @brief Disassociates a receive work request from a previous receive - * and prepares it for reuse. - * - */ -NNTI_result_t NNTI_clear_work_request ( - NNTI_work_request_t *wr); - - -/** - * @brief Disassociates a receive work request from reg_buf. - * - */ -NNTI_result_t NNTI_destroy_work_request ( - NNTI_work_request_t *wr); - - -/** - * @brief Attempts to cancel an NNTI opertion. - * - */ -NNTI_result_t NNTI_cancel ( - NNTI_work_request_t *wr); - - -/** - * @brief Attempts to cancel a list of NNTI opertions. - * - */ -NNTI_result_t NNTI_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - - -/** - * @brief Interrupts NNTI_wait*() - * - */ -NNTI_result_t NNTI_interrupt ( - const NNTI_transport_t *trans_hdl); - - -/** - * @brief Wait for a specific buffer to be acted on by a peer. - * - * \param[in] reg_buf The buffer that will be acted on by a peer. - * \param[in] remote_op The remote operation that we expect to occur. - * \param[in] timeout The amount of time to wait before giving up. - * \param[out] status The details of the completed (or timed out) operation. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status); - -/** - * @brief Wait for any buffer in the list to be acted on by a peer. - * - * \param[in] buf_list The array of buffers that will be acted on by a peer. - * \param[in] buf_count The number of buffers in the array. - * \param[in] remote_op The remote operation that we expect to occur. - * \param[in] timeout The amount of time to wait before giving up. - * \param[out] which The index of the buffer that completed. - * \param[out] status The details of the completed (or timed out) operation. - * \return A result code (NNTI_OK or an error) - * - * Caveats: All buffers in buf_list must be registered with the same transport. - */ -NNTI_result_t NNTI_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -/** - * @brief Wait for all buffers in the list to be acted on by a peer. - * - * \param[in] buf_list The array of buffers that will be acted on by a peer. - * \param[in] buf_count The number of buffers in the array. - * \param[in] remote_op The remote operation that we expect to occur. - * \param[in] timeout The amount of time to wait before giving up. - * \param[out] status The array of statuses that contain the details of the completed (or timed out) operations. - * \return A result code (NNTI_OK or an error) - * - * Caveats: All buffers in buf_list must be registered with the same transport. - */ -NNTI_result_t NNTI_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status); - -/** - * @brief Disable this transport. - * - * \param[in] trans_hdl A handle to the configured transport. - * \return A result code (NNTI_OK or an error) -*/ -NNTI_result_t NNTI_fini ( - const NNTI_transport_t *trans_hdl); - -/** @} */ - - - - -#if defined(HAVE_TRIOS_INFINIBAND) -#define NNTI_DEFAULT_TRANSPORT NNTI_TRANSPORT_IB -#elif defined(HAVE_TRIOS_GEMINI) -#define NNTI_DEFAULT_TRANSPORT NNTI_TRANSPORT_GEMINI -#elif defined(HAVE_TRIOS_BGPDCMF) -#define NNTI_DEFAULT_TRANSPORT NNTI_TRANSPORT_DCMF -#elif defined(HAVE_TRIOS_BGQPAMI) -#define NNTI_DEFAULT_TRANSPORT NNTI_TRANSPORT_PAMI -#elif defined(HAVE_TRIOS_MPI) -#define NNTI_DEFAULT_TRANSPORT NNTI_TRANSPORT_MPI -#elif defined(HAVE_TRIOS_PORTALS) || defined(HAVE_TRIOS_CRAYPORTALS) -#define NNTI_DEFAULT_TRANSPORT NNTI_TRANSPORT_PORTALS -#else -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_LOCAL -#endif - - - - -/** - * @brief Return TRUE if transport 't' has been successfully initialized. - */ -#define NNTI_TRANSPORT_INITIALIZED(t) -/** - * @brief Return TRUE if buffer 'b' has been filled by the transport. - */ -#define NNTI_BUFFER_FULL(b) -/** - * @brief Return TRUE if buffer 'b' is empty. - */ -#define NNTI_BUFFER_EMPTY(b) -/** - * @brief Return a 'char *' pointer to the payload of buffer 'b'. - */ -#define NNTI_BUFFER_C_POINTER(b) ((char *)(b)->payload) -/** - * @brief Return the size of buffer 'b' in bytes. - */ -#define NNTI_BUFFER_SIZE(b) (b)->payload_size - -/** - * @brief - * - * #define NNTI_INIT_BUFFER(b) \ - (b)->buffer_owner.url=""; \ - (b)->peer.url=""; - */ -#define NNTI_INIT_BUFFER(b) \ - memset((b)->buffer_owner.url, 0, sizeof((b)->buffer_owner.url)); \ - memset((b)->peer.url, 0, sizeof((b)->peer.url)); - -/** - * @brief - * #define NNTI_INIT_PEER(p) \ - (p)->url=""; - */ -#define NNTI_INIT_PEER(p) \ - memset((p)->url, 0, sizeof((p)->url)); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/Trios_nnti_fprint_types.h b/packages/trios/libraries/nessie/nnti/Trios_nnti_fprint_types.h deleted file mode 100644 index f4111142b4aa..000000000000 --- a/packages/trios/libraries/nessie/nnti/Trios_nnti_fprint_types.h +++ /dev/null @@ -1,112 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file Trios_nnti_fprint_types.h - * - * @brief Pretty print the different NSSI types. - * - */ - -#ifndef _TRIOS_NNTI_FPRINT_TYPES_H_ -#define _TRIOS_NNTI_FPRINT_TYPES_H_ - -#include -#include -#include "Trios_nnti_xdr.h" - - -#if defined(__STDC__) || defined(__cplusplus) - -extern const char* nnti_err_str(const int rc); - - extern void fprint_NNTI_remote_addr( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_remote_addr_t *addr); - extern void fprint_NNTI_remote_addr( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_remote_addr_t *addr); - - extern void fprint_NNTI_peer( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_peer_t *addr); - extern void fprint_NNTI_peer( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_peer_t *addr); - - extern void fprint_NNTI_buffer( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_buffer_t *addr); - extern void fprint_NNTI_buffer( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_buffer_t *addr); - - extern void fprint_NNTI_status( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_status_t *addr); - extern void fprint_NNTI_status( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_status_t *addr); - - -#else /* K&R C */ - - -#endif /* K&R C */ - -#endif diff --git a/packages/trios/libraries/nessie/nnti/Trios_nnti_xdr.x b/packages/trios/libraries/nessie/nnti/Trios_nnti_xdr.x deleted file mode 100644 index 6ad46768118c..000000000000 --- a/packages/trios/libraries/nessie/nnti/Trios_nnti_xdr.x +++ /dev/null @@ -1,865 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file nnti_xdr.x - * - * @brief XDR definitions for types used in the NNTI transport interface. - * - * @author Todd Kordenbrock (thkorde\@sandia.gov). - * - */ - -#ifdef RPC_HDR -%#include "Trios_xdr.h" -%#include -#endif - -#ifdef RPC_XDR -%#include "Trios_xdr.h" -%#include -#endif - - -/** @addtogroup nnti_types - * @{ - */ - -/** - * @brief The \ref nssi_size type is used to for ``size'' variables. - */ -typedef uint64_t NNTI_size; -typedef uint64_t NNTI_ssize; - -/** - * @brief Length of a hostname - */ -const NNTI_HOSTNAME_LEN = 32; - -/** - * @brief Length of a URL - */ -const NNTI_URL_LEN = 128; - -/** - * @brief The size of a buffer used for receiving requests (should be configurable via URL). - * - * This should be a multiple of 64 for best cache performance. 1216=64*19 - */ -const NNTI_REQUEST_BUFFER_SIZE=1216; -/** - * @brief The size of a buffer used for receiving results (should be configurable via URL). - * - * This should be a multiple of 64 for best cache performance. 1216=64*19 - */ -const NNTI_RESULT_BUFFER_SIZE=1216; - - -enum NNTI_datatype_t { - NNTI_dt_transport = 1111, - NNTI_dt_peer = 1112, - NNTI_dt_buffer = 1113, - NNTI_dt_work_request = 1114, - NNTI_dt_status = 1115 -}; - - -/** - * @brief Enumerator of the transport mechanisms supported by NNTI. - * - * The \ref NNTI_transport_id_t enumerator provides integer values - * to represent the supported transport mechanisms. - */ -enum NNTI_transport_id_t { - /** @brief No operations permitted. */ - NNTI_TRANSPORT_NULL, - - /** @brief Use Portals to transfer rpc requests. */ - NNTI_TRANSPORT_PORTALS, - - /** @brief Use Infiniband to transfer rpc requests. */ - NNTI_TRANSPORT_IB, - - /** @brief Use Cray Gemini to transfer rpc requests. */ - NNTI_TRANSPORT_GEMINI, - - /** @brief Use DCMF for Blue Gene /P transfer rpc requests. */ - NNTI_TRANSPORT_DCMF, - - /** @brief Use PAMI for Blue Gene /P transfer rpc requests. */ - NNTI_TRANSPORT_PAMI, - - /** @brief Use Cray Gemini to transfer rpc requests. */ - NNTI_TRANSPORT_MPI, - - /** @brief Use a local buffer (no remote operations). */ - NNTI_TRANSPORT_LOCAL -}; - -/** - * @brief The number of transport mechanisms supported by NNTI. - */ -const NNTI_TRANSPORT_COUNT = 8; - - -/** - * @brief Enumerator of results that NNTI functions could generate. - * - * The \ref NNTI_result_t enumerator provides integer values - * for NNTI function outcomes. - */ -enum NNTI_result_t { - /** @brief The function completed successfully. */ - NNTI_OK = 0, - - /** @brief Unspecified I/O error. */ - NNTI_EIO = 1001, - - /** @brief The size of the message is larger than the supported maximum size. */ - NNTI_EMSGSIZE, - - /** @brief The operation or process has been canceled. */ - NNTI_ECANCELED, - - /** @brief An operation timed out. */ - NNTI_ETIMEDOUT, - - /** @brief The value of the variable or parameter is invalid. */ - NNTI_EINVAL, - - /** @brief No memory is available. */ - NNTI_ENOMEM, - - /** @brief No such entry. */ - NNTI_ENOENT, - - /** @brief Unsupported operation. */ - NNTI_ENOTSUP, - - /** @brief The item already exists. */ - NNTI_EEXIST, - - /** @brief Unsuccessful RPC operation. */ - NNTI_EBADRPC, - - /** @brief Not initialized. */ - NNTI_ENOTINIT, - - /** @brief Insufficient priveleges to perform operation. */ - NNTI_EPERM, - - /** @brief Either an operation was interrupted by a signal - * or NNTI_wait() was interrupted by NNTI_interrupt(). */ - NNTI_EINTR, - - /** @brief An async operation would have blocked. */ - NNTI_EWOULDBLOCK, - - /** @brief The requested resource is temporarily unavailable. Retry later. */ - NNTI_EAGAIN, - - /** @brief The request could not be delivered. */ - NNTI_EDROPPED, - - /** @brief Error unpacking an NNTI data structure. */ - NNTI_EDECODE, - - /** @brief Error packing an NNTI data structure. */ - NNTI_EENCODE - -}; - - -/** - * atomic operations that may be implemented by a transport - */ -enum NNTI_atomic_op_t { - NNTI_ATOMIC_FADD -}; - - - -/*********** TCP/IP address types ***********/ - -/** - * @brief Binary encoding of a TCP/IP host address. - * - * The \ref NNTI_ip_addr type identifies a particular node. - */ -typedef uint32_t NNTI_ip_addr; - -/** - * @brief TCP port in NBO. - * - * The \ref NNTI_tcp_addr type identifies a particular port. - */ -typedef uint16_t NNTI_tcp_port; - - -/*********** NULL Process Types ***********/ - -/** - * @brief Remote process identifier for the NULL transport. - * - */ -struct NNTI_null_process_t { - int i; /* unused, but empty structs are not allowed */ -}; - - -/*********** Portals Process Types ***********/ - -/** - * @brief Network node identifier (used by Portals). - * - * The \ref NNTI_nid type identifies a particular node. - */ -typedef uint32_t NNTI_nid; - -/** - * @brief Process identifier (used by Portals). - * - * The \ref NNTI_pid type identifies a particular process. - */ -typedef uint32_t NNTI_pid; - -/** - * @brief Remote process identifier for Portals. - * - * The \ref NNTI_portals_process_t identifies a particular process - * on a particular node. - */ -struct NNTI_portals_process_t { - /** @brief Portals Node ID. */ - NNTI_nid nid; - /** @brief Portals Process ID. */ - NNTI_pid pid; -}; - - -/*********** IB Process Types ***********/ - -/** - * @brief The queue pair number of an IB connection. - * - * The \ref NNTI_qp_num type identifies a particular connection - * between two IB hosts. - */ -typedef uint32_t NNTI_qp_num; - -/** - * @brief Remote process identifier for IB. - * - * The \ref NNTI_ib_process_t identifies a particular process - * on a particular node. If a connection has been established to the - * represented process, then that connection is identified by 'qp_num'. - */ -struct NNTI_ib_process_t { -/* /** @brief name on the peer */ -/* string hostname; */ - /** @brief IP address encoded in Network Byte Order */ - NNTI_ip_addr addr; - /** @brief TCP port encoded in Network Byte Order */ - NNTI_tcp_port port; - /** @brief IB connection queue pair number */ - NNTI_qp_num qpn; -}; - - -/*********** Gemini Process Types ***********/ - -/** - * @brief The instance ID of a Gemini process. - * - * The \ref NNTI_inst_id type identifies a particular process - * within a communication domain. - */ -typedef uint32_t NNTI_instance_id; - -/** - * @brief Remote process identifier for Gemini. - * - * The \ref NNTI_gni_process_t identifies a particular process - * on a particular node. If a connection has been established to the - * represented process, then that connection is identified by 'inst_id'. - */ -struct NNTI_gni_process_t { - /** @brief IP address encoded in Network Byte Order */ - NNTI_ip_addr addr; - /** @brief TCP port encoded in Network Byte Order */ - NNTI_tcp_port port; - /** @brief Gemini process instance ID */ - NNTI_instance_id inst_id; -}; - - -/*********** BG/P DCMF Process Types ***********/ - -/** - * @brief Remote process identifier for BG/P DCMF. - * - * The \ref NNTI_bgpdcmf_process_t identifies a particular process - * on a particular node. - */ -struct NNTI_bgpdcmf_process_t { - int xcoord; - int ycoord; - int zcoord; - int pset_rank; -}; - - -/*********** BG/Q PAMI Process Types ***********/ - -/** - * @brief Remote process identifier for BG/Q PAMI. - * - * The \ref NNTI_bgpdcmf_process_t identifies a particular process - * on a particular node. - */ -struct NNTI_bgqpami_process_t { - int pset_rank; - int taskid; - int thrid; -}; - - -/*********** MPI Process Types ***********/ - -/** - * @brief Remote process identifier for MPI. - * - * The \ref NNTI_mpi_process_t identifies a particular process - * on a particular node. - */ -struct NNTI_mpi_process_t { - /** @brief MPI rank. */ - int rank; -}; - - -/*********** Local Process Types ***********/ - -/** - * @brief Remote process identifier for the Local transport. - * - */ -struct NNTI_local_process_t { - int i; /* unused, but empty structs are not allowed */ -}; - - -/*********** Remote Process Union ***********/ - -/** - * @brief A structure to represent a remote processes. - * - * The NNTI_remote_process_t structure contains the - * transport specific info needed to identify a process running - * on a remote node. - */ -#if defined(RPC_HDR) || defined(RPC_XDR) -union NNTI_remote_process_t switch (NNTI_transport_id_t transport_id) { - /** @brief The NULL representation of a process on the network. */ - case NNTI_TRANSPORT_NULL: NNTI_null_process_t null; - /** @brief The Portals representation of a process on the network. */ - case NNTI_TRANSPORT_PORTALS: NNTI_portals_process_t portals; - /** @brief The IB representation of a process on the network. */ - case NNTI_TRANSPORT_IB: NNTI_ib_process_t ib; - /** @brief The Cray Gemini representation of a process on the network. */ - case NNTI_TRANSPORT_GEMINI: NNTI_gni_process_t gni; - /** @brief The BGP DCMF library usage on the torus network. */ - case NNTI_TRANSPORT_DCMF: NNTI_bgpdcmf_process_t bgpdcmf; - /** @brief The BGQ PAMI library usage on the torus network. */ - case NNTI_TRANSPORT_PAMI: NNTI_bgqpami_process_t bgqpami; - /** @brief The MPI representation of a process on the network. */ - case NNTI_TRANSPORT_MPI: NNTI_mpi_process_t mpi; - /** @brief The Local representation of a process on the network. */ - case NNTI_TRANSPORT_LOCAL: NNTI_local_process_t local; -}; -#else -union NNTI_remote_process_t { - /** @brief The NULL representation of a process on the network. */ - NNTI_null_process_t null; - /** @brief The Portals representation of a process on the network. */ - NNTI_portals_process_t portals; - /** @brief The IB representation of a process on the network. */ - NNTI_ib_process_t ib; - /** @brief The Cray Gemini representation of a process on the network. */ - NNTI_gni_process_t gni; - /** @brief The BGPDMA representation of a process on the network. */ - NNTI_bgpdcmf_process_t bgpdcmf; - /** @brief The BGQPAMI representation of a process on the network. */ - NNTI_bgqpami_process_t bgqpami; - /** @brief The MPI representation of a process on the network. */ - NNTI_mpi_process_t mpi; - /** @brief The Local representation of a process on the network. */ - NNTI_local_process_t local; -}; -#endif - - -/*********** Peer Type ***********/ - -/** - * @brief Handle to an NNTI process. - * - * This is the datatype used by NNTI clients to reference another process. - * Use this handle to move data to/from the process. - */ -struct NNTI_peer_t { - NNTI_datatype_t datatype; - - /** @brief string encoding of a process on the network */ - opaque url[NNTI_URL_LEN]; - - /** @brief binary encoding of a process on the network */ - NNTI_remote_process_t peer; -}; - - -/*********** NULL RDMA Address Types ***********/ - -/** - * @brief RDMA address used for the NULL transport. - */ -struct NNTI_null_rdma_addr_t { - int i; /* unused, but empty structs are not allowed */ -}; - - -/*********** Portals RDMA Address Types ***********/ - -/** - * @brief Definition for match bits in Portals. - */ -typedef uint64_t NNTI_match_bits; - -/** - * @brief Definition for a remote buffer id. - */ -enum NNTI_portals_indices { - NNTI_REQ_PT_INDEX = 1, /* where to send requests */ - NNTI_RECV_PT_INDEX, /* where to receive data */ - NNTI_DATA_PT_INDEX, /* where to put/get data */ - NNTI_LONG_ARGS_PT_INDEX, /* where to fetch long args */ - NNTI_LONG_RES_PT_INDEX /* where to fetch long results */ -}; - -/** - * @brief RDMA address used for the Portals implementation. - */ -struct NNTI_portals_rdma_addr_t { - /** @brief The memory buffer ID. */ - NNTI_portals_indices buffer_id; - /** @brief The match bits (required by Portals). */ - NNTI_match_bits match_bits; - /** @brief Size of the the memory buffer. */ - uint32_t size; -}; - - -/*********** IB RDMA Address Types ***********/ - -/** - * @brief RDMA address used for the InfiniBand implementation. - */ -struct NNTI_ib_rdma_addr_t { - /** @brief Address of the memory buffer cast to a uint64_t. */ - uint64_t buf; - /** @brief The key that a remote processes needs to access this buffer. */ - uint32_t key; - /** @brief Size of the the memory buffer. */ - uint32_t size; - - /** @brief Address of the ACK buffer cast to a uint64_t. */ - uint64_t ack_buf; - /** @brief The key that a remote processes needs to access the ACK buffer. */ - uint32_t ack_key; - /** @brief Size of the the ACK buffer. */ - uint32_t ack_size; -}; - - -/*********** Gemini RDMA Address Types ***********/ - -enum NNTI_gni_buffer_type_t { - NNTI_GNI_RDMA_INITIATOR, - NNTI_GNI_RDMA_TARGET, - NNTI_GNI_SEND_SRC, - NNTI_GNI_REQUEST_BUFFER -}; - -struct NNTI_gni_mem_hdl_t { - uint64_t qword1; - uint64_t qword2; -}; - -/** - * @brief RDMA address used for the Gemini implementation. - */ -struct NNTI_gni_rdma_addr_t { - /** @brief Address of the memory buffer cast to a uint64_t. */ - uint64_t buf; - /** @brief Size of the the memory buffer. */ - uint32_t size; - /** @brief The key that a remote processes needs to access this buffer. */ - NNTI_gni_mem_hdl_t mem_hdl; - - /** @brief Address of the Work Completion buffer cast to a uint64_t. */ - uint64_t wc_addr; - /** @brief The key that a remote processes needs to access the Work Completion buffer. */ - NNTI_gni_mem_hdl_t wc_mem_hdl; - - /** @brief Identifies the type of buffer (FMA Short Msg or RDMA) */ - NNTI_gni_buffer_type_t type; -}; - - -/*********** BG/P DCMF RDMA Address Types ***********/ - -struct NNTI_bgpdcmf_memreg_hdl_t { - unsigned word0; - unsigned word1; - unsigned word2; - unsigned word3; -}; - -enum NNTI_bgpdcmf_buffer_type_t { - NNTI_DCMF_REQUEST_BUFFER, - NNTI_DCMF_RESULT_BUFFER, - NNTI_DCMF_SEND_SRC, - NNTI_DCMF_RECEIVE_DST -}; - -/** - * @brief RDMA address used for the BG/P DCMF implementation. - */ -struct NNTI_bgpdcmf_rdma_addr_t { - /** @brief Address of the memory buffer cast to a uint64_t. */ - uint64_t buf; - /** @brief Size of the the memory buffer. */ - uint32_t size; - uint32_t owner_rank; - NNTI_bgpdcmf_buffer_type_t type; - NNTI_bgpdcmf_memreg_hdl_t mem_hdl; - uint64_t wc_addr; - NNTI_bgpdcmf_memreg_hdl_t wc_mem_hdl; -}; - - -/*********** BG/Q PAMI RDMA Address Types ***********/ - -enum NNTI_bgqpami_buffer_type_t { - NNTI_PAMI_REQUEST_BUFFER, - NNTI_PAMI_RESULT_BUFFER, - NNTI_PAMI_SEND_SRC, - NNTI_PAMI_RECEIVE_DST -}; - -struct NNTI_bgqpami_rdma_addr_t { - uint64_t buf; - uint32_t size; - uint32_t owner_rank; - NNTI_bgqpami_buffer_type_t type; - uint64_t mem_hdl; - uint64_t wc_addr; - uint64_t wc_mem_hdl; -}; - - -/*********** MPI RDMA Address Types ***********/ - -/** - * @brief RDMA address used for the MPI implementation. - */ -struct NNTI_mpi_rdma_addr_t { - /** @brief The MPI tag for RTR/RTS msg. */ - NNTI_match_bits cmd_tag; - /** @brief The MPI tag for GET data msg. */ - NNTI_match_bits get_data_tag; - /** @brief The MPI tag for PUT data msg. */ - NNTI_match_bits put_data_tag; - /** @brief Size of the the memory buffer. */ - uint32_t size; -}; - - -/*********** Local RDMA Address Types ***********/ - -/** - * @brief RDMA address used for the Local transport. - */ -struct NNTI_local_rdma_addr_t { - int i; /* unused, but empty structs are not allowed */ -}; - - -/*********** Remote Address Union ***********/ - -/** - * @brief A structure to represent a remote memory region. - * - * The NNTI_remote_addr_t structure contains the - * transport specific info needed to identify a memory region - * on a remote node. - */ -#if defined(RPC_HDR) || defined(RPC_XDR) -union NNTI_remote_addr_t switch (NNTI_transport_id_t transport_id) { - /** @brief The NULL representation of a memory region. */ - case NNTI_TRANSPORT_NULL: NNTI_null_rdma_addr_t null; - /** @brief The Portals representation of a memory region. */ - case NNTI_TRANSPORT_PORTALS: NNTI_portals_rdma_addr_t portals; - /** @brief The IB representation of a memory region. */ - case NNTI_TRANSPORT_IB: NNTI_ib_rdma_addr_t ib; - /** @brief The Cray Gemini representation of a memory region. */ - case NNTI_TRANSPORT_GEMINI: NNTI_gni_rdma_addr_t gni; - /** @brief The BGP DCMF representation of a memory region. */ - case NNTI_TRANSPORT_DCMF: NNTI_bgpdcmf_rdma_addr_t bgpdcmf; - /** @brief The BGQ PAMI representation of a memory region. */ - case NNTI_TRANSPORT_PAMI: NNTI_bgqpami_rdma_addr_t bgqpami; - /** @brief The MPI representation of a memory region. */ - case NNTI_TRANSPORT_MPI: NNTI_mpi_rdma_addr_t mpi; - /** @brief The Local representation of a memory region. */ - case NNTI_TRANSPORT_LOCAL: NNTI_local_rdma_addr_t local; -}; -#else -union NNTI_remote_addr_t { - /** @brief The NULL representation of a memory region. */ - NNTI_null_rdma_addr_t null; - /** @brief The Portals representation of a memory region. */ - NNTI_portals_rdma_addr_t portals; - /** @brief The IB representation of a memory region. */ - NNTI_ib_rdma_addr_t ib; - /** @brief The Cray Gemini representation of a memory region. */ - NNTI_gni_rdma_addr_t gni; - /** @brief The BGP DCMF representation of a memory region. */ - NNTI_bgpdcmf_rdma_addr_t bgpdcmf; - /** @brief The BGQ PAMI representation of a memory region. */ - NNTI_bgqpami_rdma_addr_t bgqpami; - /** @brief The MPI representation of a memory region. */ - NNTI_mpi_rdma_addr_t mpi; - /** @brief The Local representation of a memory region. */ - NNTI_local_rdma_addr_t local; -}; -#endif - -/** - * @brief Array of 16-byte structures that we can send with the request. - * - * Rpcgen will use this definition to define encoding functions to - * encode and decode an array of \ref data_t structures. We will - * use these functions when sending the array with the request. - */ -typedef NNTI_remote_addr_t NNTI_remote_addr_array_t<>; - - - -/*********** Buffer Operations Type ***********/ - -/** - * @brief The operations that are permitted on a buffer - * - * The NNTI_buf_ops_t enum defines the operations that can be - * performed on a buffer. These operations can be OR'd together to create - * a multipurpose buffer. - */ -enum NNTI_buf_ops_t { - /** @brief a local process/NIC can read from this buffer */ - NNTI_BOP_LOCAL_READ=1, - /** @brief a remote process/NIC can read from this buffer */ - NNTI_BOP_REMOTE_READ=2, - /** @brief a local process/NIC can write to this buffer */ - NNTI_BOP_LOCAL_WRITE=4, - /** @brief a remote process/NIC can write to this buffer */ - NNTI_BOP_REMOTE_WRITE=8, - /** @brief operations on this memory generate events that cause work requests to change state */ - NNTI_BOP_WITH_EVENTS=16, - /** @brief SENDs to this memory occur at the offset+length of the last SEND */ - NNTI_BOP_QUEUING=32, - /** @brief this buffer has multiple receive slots */ - NNTI_BOP_RECV_QUEUE=64, - /** @brief this buffer allows atomic operations */ - NNTI_BOP_ATOMICS=128 -}; - -/** @brief this buffer can be put from */ -%#define NNTI_PUT_SRC ((NNTI_buf_ops_t)(NNTI_BOP_LOCAL_READ|NNTI_BOP_WITH_EVENTS)) -/** @brief this buffer can be put into */ -%#define NNTI_PUT_DST ((NNTI_buf_ops_t)(NNTI_BOP_REMOTE_WRITE)) -/** @brief this buffer can be got from */ -%#define NNTI_GET_SRC ((NNTI_buf_ops_t)(NNTI_BOP_REMOTE_READ)) -/** @brief this buffer can be got into */ -%#define NNTI_GET_DST ((NNTI_buf_ops_t)(NNTI_BOP_LOCAL_WRITE|NNTI_BOP_WITH_EVENTS)) -/** @brief this buffer can be sent from */ -%#define NNTI_SEND_SRC ((NNTI_buf_ops_t)(NNTI_BOP_LOCAL_READ|NNTI_BOP_WITH_EVENTS)) -/** @brief this buffer can be received into */ -%#define NNTI_RECV_DST ((NNTI_buf_ops_t)(NNTI_BOP_REMOTE_WRITE|NNTI_BOP_WITH_EVENTS)) -/** @brief this buffer has multiple receive slots */ -%#define NNTI_RECV_QUEUE ((NNTI_buf_ops_t)(NNTI_BOP_RECV_QUEUE)) -/** @brief this buffer allows atomic operations */ -%#define NNTI_ATOMICS ((NNTI_buf_ops_t)(NNTI_BOP_ATOMICS)) - - -/*********** Buffer Type ***********/ - -/** - * @brief handle to a memory buffer prepared by NNTI_register_memory - * - * The NNTI_buffer_t structure contains the - * location of a buffer on the network. This is all the info - * a peer needs to put/get this buffer. - */ -struct NNTI_buffer_t { - NNTI_datatype_t datatype; - - /** @brief the transport where this buffer is registered */ - NNTI_transport_id_t transport_id; - - /** @brief the process in which this buffer resides */ - NNTI_peer_t buffer_owner; - - /** @brief Array of remote addresses. Each remote address is a segment of the complete buffer. */ - NNTI_remote_addr_array_t buffer_segments; - - /** @brief permitted operations */ - NNTI_buf_ops_t ops; - - /** @brief Size of this buffer. */ - uint64_t payload_size; - /** @brief Local address of the memory buffer cast to a uint64_t. */ - uint64_t payload; - - /** @brief Private storage (cast to a uint64_t). */ - uint64_t transport_private; -}; - - -/*********** Work Request Types ***********/ - -/** - * @brief handle to a work request prepared by an operation initiator - * - */ -struct NNTI_work_request_t { - NNTI_datatype_t datatype; - - /** @brief the transport where this buffer is registered */ - NNTI_transport_id_t transport_id; - - /** @brief */ - NNTI_buffer_t *reg_buf; - - /** @brief permitted operations */ - NNTI_buf_ops_t ops; - - /** @brief operation result */ - NNTI_result_t result; - - /** @brief Private storage (cast to a uint64_t). */ - uint64_t transport_private; -}; - - -/*********** Transport Header Type ***********/ - -/** - * @brief A header that is prepended to all requests and results. - * - */ -struct NNTI_transport_header_t { - /** @brief a piece of magic to describe the type of message contained in the payload */ - uint32_t message_magic; - /** @brief some identifier the transport can use to create a thread of messages */ - uint32_t exchange_id; -}; - - -/*********** Status Type ***********/ - -/** - * @brief The status of an NNTI operation. - * - * The NNTI_status_t structure contains the result of an NNTI buffer - * operation. The result and op fields will always contain reasonable values. - * If the operation was successful (result==NNTI_OK), then the remaining - * fields will also contain reasonable values. If the operation was - * unsuccessful, then the remaining fields are undefined. - */ -struct NNTI_status_t { - NNTI_datatype_t datatype; - - /** @brief The operation performed on the buffer. */ - NNTI_buf_ops_t op; - /** @brief The result code for this operation. */ - NNTI_result_t result; - - /** @brief The address of the local buffer used by this operation. */ - uint64_t start; - /** @brief The offset into the local buffer used by this operation. */ - uint64_t offset; - /** @brief The number of bytes used by this operation. */ - uint64_t length; - - /** @brief The peer that was the data source for this operation. */ - NNTI_peer_t src; - /** @brief The peer that was the data destination for this operation. */ - NNTI_peer_t dest; -}; - - -/*********** Transport Type ***********/ - -/** - * @brief The external representation of a configured transport. - */ -struct NNTI_transport_t { - NNTI_datatype_t datatype; - - /** @brief The transport id. */ - NNTI_transport_id_t id; - /** @brief A reference to my process that can be sent to a peer so the peer can contact me. */ - NNTI_peer_t me; -}; - - -/** @} */ diff --git a/packages/trios/libraries/nessie/nnti/buffer_queue.cpp b/packages/trios/libraries/nessie/nnti/buffer_queue.cpp deleted file mode 100644 index e2563e55c6a0..000000000000 --- a/packages/trios/libraries/nessie/nnti/buffer_queue.cpp +++ /dev/null @@ -1,273 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include "Trios_config.h" - -#include "Trios_logger.h" -#include "buffer_queue.h" - -#include "assert.h" -#include "string.h" - -log_level bq_debug_level = LOG_UNDEFINED; - - -static NNTI_result_t create_buffer( - const NNTI_transport_t *trans_hdl, - const NNTI_buf_ops_t op, - const uint32_t buffer_size, - NNTI_buffer_t **buffer) -{ - NNTI_result_t nnti_rc; -// char *b; - - log_debug(bq_debug_level, "enter"); - -// b=(char *)malloc(buffer_size); -// assert(b); -// memset(b, 0, buffer_size); - *buffer=(NNTI_buffer_t *)malloc(sizeof(NNTI_buffer_t)); - assert(*buffer); - nnti_rc=NNTI_alloc( - trans_hdl, - buffer_size, - 1, - op, - *buffer); - if (nnti_rc != NNTI_OK) { - log_error(bq_debug_level, "failed registering queue buffer: %d", nnti_rc); - } - - log_debug(bq_debug_level, "exit"); - - return(nnti_rc); -} - -static NNTI_result_t destroy_buffer(NNTI_buffer_t **buffer) -{ - NNTI_result_t nnti_rc; -// char *b; - - log_debug(bq_debug_level, "enter"); - -// b=NNTI_BUFFER_C_POINTER(*buffer); - nnti_rc=NNTI_free(*buffer); - if (nnti_rc != NNTI_OK) { - log_error(bq_debug_level, "failed unregistering queue buffer: %d", nnti_rc); - } - free(*buffer); - *buffer=NULL; -// free(b); - - log_debug(bq_debug_level, "exit"); - - return(nnti_rc); -} - -int trios_buffer_queue_init( - trios_buffer_queue_t *bq, - uint32_t initial_size, - uint32_t max_size, - uint8_t create_on_fly, - NNTI_transport_t *trans_hdl, - NNTI_buf_ops_t op, - uint32_t buffer_size) -{ - NNTI_result_t nnti_rc=NNTI_OK; - NNTI_buffer_t *buffer=NULL; - - log_debug(bq_debug_level, "enter"); - - if (buffer_size <= 0) { - log_debug(bq_debug_level, "buffer_size <= 0 - queue not created"); - return((int)nnti_rc); - } - - nthread_lock_init(&bq->mutex); - - if (nthread_lock(&bq->mutex)) log_warn(bq_debug_level, "failed to get thread lock"); - - bq->current_size=0; - bq->initial_size=initial_size; - bq->max_size=max_size; - bq->create_on_fly=create_on_fly; - bq->trans_hdl=trans_hdl; - bq->op=op; - bq->buffer_size=buffer_size; - - for (uint32_t i=0;iinitial_size;i++) { - log_debug(bq_debug_level, "creating queue buffer"); - nnti_rc=create_buffer( - bq->trans_hdl, - bq->op, - bq->buffer_size, - &buffer); - if (nnti_rc==NNTI_OK) { - log_debug(bq_debug_level, "pushing queue buffer"); - bq->queue.push_back(buffer); - bq->current_size++; - } else { - log_error(bq_debug_level, "failed creating queue buffer: %d", nnti_rc); - break; - } - } - nthread_unlock(&bq->mutex); - - log_debug(bq_debug_level, "exit"); - - return((int)nnti_rc); -} - -NNTI_buffer_t *trios_buffer_queue_pop( - trios_buffer_queue_t *bq) -{ - NNTI_result_t nnti_rc=NNTI_OK; - NNTI_buffer_t *buffer=NULL; - - log_debug(bq_debug_level, "enter"); - - if (nthread_lock(&bq->mutex)) log_warn(bq_debug_level, "failed to get thread lock"); - if (!bq->queue.empty()) { - log_debug(bq_debug_level, "getting buffer from queue"); - buffer=bq->queue.front(); - bq->queue.pop_front(); - } else { - if (bq->current_size < bq->max_size) { - nnti_rc=create_buffer( - bq->trans_hdl, - bq->op, - bq->buffer_size, - &buffer); - if (nnti_rc==NNTI_OK) { - log_debug(bq_debug_level, "expanding buffer queue"); - bq->current_size++; - } else { - log_error(bq_debug_level, "failed creating queue buffer to expand the queue: %d", nnti_rc); - } - } else if ((bq->current_size == bq->max_size) && (bq->create_on_fly==TRUE)) { - nnti_rc=create_buffer( - bq->trans_hdl, - bq->op, - bq->buffer_size, - &buffer); - if (nnti_rc!=NNTI_OK) { - log_error(bq_debug_level, "failed creating on the fly queue buffer: %d", nnti_rc); - buffer=NULL; - } else { - log_debug(bq_debug_level, "creating on the fly queue buffer"); - } - } - } - nthread_unlock(&bq->mutex); - - log_debug(bq_debug_level, "exit"); - - return(buffer); -} - -void trios_buffer_queue_push( - trios_buffer_queue_t *bq, - NNTI_buffer_t *buffer) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(bq_debug_level, "enter"); - - if (nthread_lock(&bq->mutex)) log_warn(bq_debug_level, "failed to get lock"); - if (bq->queue.size() < bq->max_size) { - /* when buffers are popped, the size could be reduced to avoid transferring more bytes than necessary */ - log_debug(bq_debug_level, "reset buffer size to bq->buffer_size"); - NNTI_BUFFER_SIZE(buffer)=bq->buffer_size; - log_debug(bq_debug_level, "returning buffer to queue"); - bq->queue.push_front(buffer); - } else { - nnti_rc=destroy_buffer(&buffer); - if (nnti_rc!=NNTI_OK) { - log_error(bq_debug_level, "failed destroying on the fly queue buffer: %d", nnti_rc); - } else { - log_debug(bq_debug_level, "destroying on the fly queue buffer"); - } - } - nthread_unlock(&bq->mutex); - - log_debug(bq_debug_level, "exit"); -} - -int trios_buffer_queue_fini( - trios_buffer_queue_t *bq) -{ - NNTI_result_t nnti_rc=NNTI_OK; - NNTI_buffer_t *buffer=NULL; - - log_debug(bq_debug_level, "enter"); - - if (nthread_lock(&bq->mutex)) log_warn(bq_debug_level, "failed to get lock"); - if (bq->queue.size() != bq->current_size) { - log_warn(bq_debug_level, "buffer queue (%p) has missing entries (bq->queue.size(%llu) != bq->current_size(%llu))", - bq, (uint64_t)bq->queue.size(), (uint64_t)bq->current_size); - } - if (bq->queue.size() < bq->initial_size) { - log_warn(bq_debug_level, "buffer queue (%p) has missing entries (bq->queue.size(%llu) < bq->initial_size(%llu))", - bq, (uint64_t)bq->queue.size(), (uint64_t)bq->initial_size); - } - if (bq->queue.size() > bq->max_size) { - log_warn(bq_debug_level, "buffer queue (%p) has extra entries (bq->queue.size(%llu) > bq->max_size(%llu))", - bq, (uint64_t)bq->queue.size(), (uint64_t)bq->max_size); - } - while (!bq->queue.empty()) { - buffer=bq->queue.front(); - bq->queue.pop_front(); - nnti_rc=destroy_buffer(&buffer); - if (nnti_rc!=NNTI_OK) { - log_error(bq_debug_level, "failed destroying queue buffer: %d", nnti_rc); - break; - } - } - nthread_unlock(&bq->mutex); - - nthread_lock_fini(&bq->mutex); - - log_debug(bq_debug_level, "exit"); - - return((int)nnti_rc); -} diff --git a/packages/trios/libraries/nessie/nnti/buffer_queue.h b/packages/trios/libraries/nessie/nnti/buffer_queue.h deleted file mode 100644 index 5da22cfa930a..000000000000 --- a/packages/trios/libraries/nessie/nnti/buffer_queue.h +++ /dev/null @@ -1,108 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file buffer_queue.h - * - * @brief API for a circular list of NNTI_buffer_t. - * - * @author Todd Kordenbrock (thkorde\@sandia.gov). - * - */ - -#ifndef _TRIOS_BUFFER_QUEUE_H_ -#define _TRIOS_BUFFER_QUEUE_H_ - - -#include "Trios_config.h" - -#include "Trios_threads.h" - -#include "Trios_nnti.h" - -#include - -typedef std::deque buffer_queue_t; - -typedef struct trios_buffer_queue { - nthread_lock_t mutex; - buffer_queue_t queue; - uint32_t current_size; - uint32_t initial_size; - uint32_t max_size; - uint8_t create_on_fly; - NNTI_transport_t *trans_hdl; - NNTI_buf_ops_t op; - uint32_t buffer_size; -} trios_buffer_queue_t; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - extern int trios_buffer_queue_init( - trios_buffer_queue_t *bq, - uint32_t initial_size, - uint32_t max_size, - uint8_t create_on_fly, - NNTI_transport_t *trans_hdl, - NNTI_buf_ops_t op, - uint32_t buffer_size); - extern NNTI_buffer_t *trios_buffer_queue_pop( - trios_buffer_queue_t *bq); - extern void trios_buffer_queue_push( - trios_buffer_queue_t *bq, - NNTI_buffer_t *buffer); - extern int trios_buffer_queue_fini( - trios_buffer_queue_t *bq); - -#endif - - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/nessie/nnti/cmake/Dependencies.cmake b/packages/trios/libraries/nessie/nnti/cmake/Dependencies.cmake deleted file mode 100644 index eb56e56d691b..000000000000 --- a/packages/trios/libraries/nessie/nnti/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triossupport) -SET(LIB_OPTIONAL_DEP_PACKAGES Trioscommsplitter) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS CrayPortals Portals Gemini InfiniBand BGPDCMF HPCToolkit) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/libraries/nessie/nnti/fprint_types.cpp b/packages/trios/libraries/nessie/nnti/fprint_types.cpp deleted file mode 100644 index 4c700e6fc46a..000000000000 --- a/packages/trios/libraries/nessie/nnti/fprint_types.cpp +++ /dev/null @@ -1,480 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file fprint_types.c - * - * @brief Pretty print the different NSSI types. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 1640 $. - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $. - * - */ - -#include -#include /* find memcpy() */ -#include -#include - -#include "Trios_logger.h" -#include "Trios_nnti_xdr.h" -#include "Trios_nnti_fprint_types.h" - -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE -#endif -#include - - - -/* ----------- Utility functions to output types ---- */ - - - -static const char *myitoa(int val) { - static char buf[32]; - - snprintf(buf, sizeof(buf), "%d", val); - return buf; -} - -/** - * @brief Output a string associated with a return code. - * - * @param rc @input the return code. - * - * @returns A string associated with the return code. - */ -const char *nnti_err_str(int rc) -{ - switch (rc) { - - case NNTI_OK: - return "NNTI_OK"; - - /** @brief Unspecified I/O error. */ - case NNTI_EIO: - return "NNTI_EIO"; - - /** @brief The size of the message is larger than the supported maximum size. */ - case NNTI_EMSGSIZE: - return "NNTI_EMSGSIZE"; - - /** @brief The operation or process has been canceled. */ - case NNTI_ECANCELED: - return "NNTI_ECANCELED"; - - /** @brief An operation timed out. */ - case NNTI_ETIMEDOUT: - return "NNTI_ETIMEDOUT"; - - /** @brief The value of the variable or parameter is invalid. */ - case NNTI_EINVAL: - return "NNTI_EINVAL"; - - /** @brief No memory is available. */ - case NNTI_ENOMEM: - return "NNTI_ENOMEM"; - - /** @brief No such entry. */ - case NNTI_ENOENT: - return "NNTI_ENOENT"; - - /** @brief Unsupported operation. */ - case NNTI_ENOTSUP: - return "NNTI_ENOTSUP"; - - /** @brief The item already exists. */ - case NNTI_EEXIST: - return "NNTI_EEXIST"; - - /** @brief Unsuccessful RPC operation. */ - case NNTI_EBADRPC: - return "NNTI_EBADRPC"; - - /** @brief Not initialized. */ - case NNTI_ENOTINIT: - return "NNTI_ENOTINIT"; - - /** @brief Insufficient priveleges to perform operation. */ - case NNTI_EPERM: - return "NNTI_EPERM"; - - default: - return myitoa(rc); - } -} - - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_remote_addr( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_remote_addr_t *addr) -{ - char subprefix[100]; - - snprintf(subprefix, 100, "%s ", prefix); - - if (addr == NULL) { - out << prefix << " " << name << "= NULL" << std::endl; - return; - } - - log_debug(LOG_ALL, "remote_addr=%p", addr); - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - out << subprefix << " transport_id = " << addr->transport_id << std::endl; - switch (addr->transport_id) { - case NNTI_TRANSPORT_PORTALS: - out << subprefix << " buffer_id = " << addr->NNTI_remote_addr_t_u.portals.buffer_id << std::endl; - out << subprefix << " match_bits = " << addr->NNTI_remote_addr_t_u.portals.match_bits << std::endl; - out << subprefix << " size = " << addr->NNTI_remote_addr_t_u.portals.size << std::endl; - break; - case NNTI_TRANSPORT_IB: - out << subprefix << " buf = " << addr->NNTI_remote_addr_t_u.ib.buf << std::endl; - out << subprefix << " key = " << addr->NNTI_remote_addr_t_u.ib.key << std::endl; - out << subprefix << " size = " << addr->NNTI_remote_addr_t_u.ib.size << std::endl; - out << subprefix << " ack_buf = " << addr->NNTI_remote_addr_t_u.ib.ack_buf << std::endl; - out << subprefix << " ack_size = " << addr->NNTI_remote_addr_t_u.ib.ack_size << std::endl; - break; - case NNTI_TRANSPORT_GEMINI: - out << subprefix << " type = " << addr->NNTI_remote_addr_t_u.gni.type << std::endl; - out << subprefix << " buf = " << addr->NNTI_remote_addr_t_u.gni.buf << std::endl; - out << subprefix << " mem_hdl = (" - << addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword1 << "," - << addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword2 << ")" << std::endl; - out << subprefix << " size = " << addr->NNTI_remote_addr_t_u.gni.size << std::endl; - out << subprefix << " wc_addr = " << addr->NNTI_remote_addr_t_u.gni.wc_addr << std::endl; - out << subprefix << " wc_mem_hdl = (" - << addr->NNTI_remote_addr_t_u.gni.wc_mem_hdl.qword1 << "," - << addr->NNTI_remote_addr_t_u.gni.wc_mem_hdl.qword2 << ")" << std::endl; - break; - case NNTI_TRANSPORT_MPI: - out << subprefix << " cmd_tag = " << addr->NNTI_remote_addr_t_u.mpi.cmd_tag << std::endl; - out << subprefix << " get_data_tag = " << addr->NNTI_remote_addr_t_u.mpi.get_data_tag << std::endl; - out << subprefix << " put_data_tag = " << addr->NNTI_remote_addr_t_u.mpi.put_data_tag << std::endl; - out << subprefix << " size = " << addr->NNTI_remote_addr_t_u.mpi.size << std::endl; - break; - case NNTI_TRANSPORT_LOCAL: - case NNTI_TRANSPORT_NULL: - break; - } - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_remote_addr( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_remote_addr_t *addr) -{ - std::stringstream out(std::stringstream::out); - - fprint_NNTI_remote_addr(out, name, prefix, addr); - - fprintf(fp, "%s", out.str().c_str()); -} - - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_remote_addr_array( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_remote_addr_array_t *addr_array) -{ - char subprefix[100]; - - snprintf(subprefix, 100, "%s ", prefix); - - if (addr_array == NULL) { - out << prefix << " " << name << "= NULL" << std::endl; - return; - } - - log_debug(LOG_ALL, "addr_array->NNTI_remote_addr_array_t_len=%d", addr_array->NNTI_remote_addr_array_t_len); - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - for (int i=0;iNNTI_remote_addr_array_t_len;i++) { - fprint_NNTI_remote_addr(out, "buffer_addr", subprefix, &addr_array->NNTI_remote_addr_array_t_val[i]); - } - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_remote_addr_array( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_remote_addr_array_t *addr_array) -{ - std::stringstream out(std::stringstream::out); - - fprint_NNTI_remote_addr_array(out, name, prefix, addr_array); - - fprintf(fp, "%s", out.str().c_str()); -} - - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_peer( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_peer_t *addr) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - if (addr == NULL) { - out << prefix << " " << name << " = NULL " << std::endl; - return; - } - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - out << subprefix << " url = " << addr->url << std::endl; - out << subprefix << " transport_id = " << (int)addr->peer.transport_id << std::endl; - switch (addr->peer.transport_id) { - case NNTI_TRANSPORT_PORTALS: - out << subprefix << " nid = " << addr->peer.NNTI_remote_process_t_u.portals.nid << std::endl; - out << subprefix << " pid = " << addr->peer.NNTI_remote_process_t_u.portals.pid << std::endl; - break; - case NNTI_TRANSPORT_IB: - out << subprefix << " addr = " << addr->peer.NNTI_remote_process_t_u.ib.addr << std::endl; - out << subprefix << " port = " << addr->peer.NNTI_remote_process_t_u.ib.port << std::endl; - out << subprefix << " qpn = " << addr->peer.NNTI_remote_process_t_u.ib.qpn << std::endl; - break; - case NNTI_TRANSPORT_GEMINI: - out << subprefix << " addr = " << addr->peer.NNTI_remote_process_t_u.gni.addr << std::endl; - out << subprefix << " port = " << addr->peer.NNTI_remote_process_t_u.gni.port << std::endl; - out << subprefix << " inst_id = " << addr->peer.NNTI_remote_process_t_u.gni.inst_id << std::endl; - break; - case NNTI_TRANSPORT_MPI: - out << subprefix << " rank = " << addr->peer.NNTI_remote_process_t_u.mpi.rank << std::endl; - break; - default: - break; - } - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_peer( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_peer_t *addr) -{ - std::stringstream out(std::stringstream::out); - - fprint_NNTI_peer(out, name, prefix, addr); - - fprintf(fp, "%s", out.str().c_str()); -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_buffer( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_buffer_t *addr) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - if (addr == NULL) { - out << prefix << " " << name << " = NULL " << std::endl; - return; - } - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - out << subprefix << " transport_id = " << addr->transport_id << std::endl; - fprint_NNTI_peer(out, "buffer_owner", subprefix.c_str(), &addr->buffer_owner); - fprint_NNTI_remote_addr_array(out, "buffer_segments", subprefix.c_str(), &addr->buffer_segments); - out << subprefix << " ops = " << addr->ops << std::endl; - out << subprefix << " payload_size = " << addr->payload_size << std::endl; - out << subprefix << " payload = " << addr->payload << std::endl; - out << subprefix << " transport_private = " << addr->transport_private << std::endl; - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_buffer( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_buffer_t *addr) -{ - std::stringstream out(std::stringstream::out); - - fprint_NNTI_buffer(out, name, prefix, addr); - - fprintf(fp, "%s", out.str().c_str()); -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_status( - std::ostream &out, - const char *name, - const char *prefix, - const NNTI_status_t *status) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - if (status == NULL) { - out << prefix << " " << name << " = NULL " << std::endl; - return; - } - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - out << subprefix << " op = " << status->op << std::endl; - out << subprefix << " result = " << status->result << std::endl; - out << subprefix << " start = " << status->start << std::endl; - out << subprefix << " offset = " << status->offset << std::endl; - out << subprefix << " length = " << status->length << std::endl; - fprint_NNTI_peer(out, "src", subprefix.c_str(), &status->src); - fprint_NNTI_peer(out, "dest", subprefix.c_str(), &status->dest); - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a Portals remote memory descriptor. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param addr The remote memory address. - */ -void fprint_NNTI_status( - FILE *fp, - const char *name, - const char *prefix, - const NNTI_status_t *status) -{ - std::stringstream out(std::stringstream::out); - - fprint_NNTI_status(out, name, prefix, status); - - fprintf(fp, "%s", out.str().c_str()); -} diff --git a/packages/trios/libraries/nessie/nnti/nnti.c b/packages/trios/libraries/nessie/nnti/nnti.c deleted file mode 100644 index 2c7dcb6b62c8..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti.c +++ /dev/null @@ -1,1353 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file nnti.c - * - * @brief nnti.c - * - * @author Todd Kordenbrock (thkorde\@sandia.gov). - * Created on: Jan 13, 2011 - */ - -#include "Trios_config.h" - -#include - -#include "Trios_nnti.h" -#include "nnti_internal.h" - -#if defined(HAVE_TRIOS_PORTALS) || defined(HAVE_TRIOS_CRAYPORTALS) -#include "nnti_ptls.h" -#endif -#if defined(HAVE_TRIOS_INFINIBAND) -#include "nnti_ib.h" -#endif -#if defined(HAVE_TRIOS_GEMINI) -#include "nnti_gni.h" -#endif -#if defined(HAVE_TRIOS_BGPDCMF) -#include "nnti_dcmf.h" -#endif -#if defined(HAVE_TRIOS_BGQPAMI) -#include "nnti_pami.h" -#endif -#if defined(HAVE_TRIOS_MPI) -#include "nnti_mpi.h" -#endif - -#include "Trios_logger.h" - - - -/* set to LOG_UNDEFINED -- log commands will use default level */ -log_level nnti_debug_level = LOG_UNDEFINED; - - -static NNTI_internal_transport_t available_transports[NNTI_TRANSPORT_COUNT]; - - -/** - * @brief Initialize NNTI to use a specific transport. - * - * Enable the use of a particular transport by this process. my_url - * allows the process to have some control (if possible) over the - * URL assigned for the transport. For example, a Portals URL to put - * might be "ptl://-1,128". This would tell Portals to use the default - * network ID, but use PID=128. If the transport - * can be initialized without this info (eg. a Portals client), my_url can - * be NULL or empty. - * - */ -NNTI_result_t NNTI_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl) -{ - NNTI_result_t rc=NNTI_OK; - static int8_t first_init=TRUE; - - if (first_init==TRUE) { - memset(&available_transports[0], 0, NNTI_TRANSPORT_COUNT*sizeof(NNTI_internal_transport_t)); - first_init=FALSE; - } - -#if defined(HAVE_TRIOS_PORTALS) || defined(HAVE_TRIOS_CRAYPORTALS) - if (trans_id == NNTI_TRANSPORT_PORTALS) { - available_transports[trans_id].initialized = 1; - available_transports[trans_id].ops.nnti_init_fn = NNTI_ptl_init; - available_transports[trans_id].ops.nnti_get_url_fn = NNTI_ptl_get_url; - available_transports[trans_id].ops.nnti_connect_fn = NNTI_ptl_connect; - available_transports[trans_id].ops.nnti_disconnect_fn = NNTI_ptl_disconnect; - available_transports[trans_id].ops.nnti_alloc_fn = NNTI_ptl_alloc; - available_transports[trans_id].ops.nnti_free_fn = NNTI_ptl_free; - available_transports[trans_id].ops.nnti_register_memory_fn = NNTI_ptl_register_memory; - available_transports[trans_id].ops.nnti_register_segments_fn = NNTI_ptl_register_segments; - available_transports[trans_id].ops.nnti_unregister_memory_fn = NNTI_ptl_unregister_memory; - available_transports[trans_id].ops.nnti_send_fn = NNTI_ptl_send; - available_transports[trans_id].ops.nnti_put_fn = NNTI_ptl_put; - available_transports[trans_id].ops.nnti_get_fn = NNTI_ptl_get; - available_transports[trans_id].ops.nnti_scatter_fn = NNTI_ptl_scatter; - available_transports[trans_id].ops.nnti_gather_fn = NNTI_ptl_gather; - available_transports[trans_id].ops.nnti_atomic_set_callback_fn = NNTI_ptl_atomic_set_callback; - available_transports[trans_id].ops.nnti_atomic_read_fn = NNTI_ptl_atomic_read; - available_transports[trans_id].ops.nnti_atomic_fop_fn = NNTI_ptl_atomic_fop; - available_transports[trans_id].ops.nnti_atomic_cswap_fn = NNTI_ptl_atomic_cswap; - available_transports[trans_id].ops.nnti_create_work_request_fn = NNTI_ptl_create_work_request; - available_transports[trans_id].ops.nnti_clear_work_request_fn = NNTI_ptl_clear_work_request; - available_transports[trans_id].ops.nnti_destroy_work_request_fn = NNTI_ptl_destroy_work_request; - available_transports[trans_id].ops.nnti_cancel_fn = NNTI_ptl_cancel; - available_transports[trans_id].ops.nnti_cancelall_fn = NNTI_ptl_cancelall; - available_transports[trans_id].ops.nnti_interrupt_fn = NNTI_ptl_interrupt; - available_transports[trans_id].ops.nnti_wait_fn = NNTI_ptl_wait; - available_transports[trans_id].ops.nnti_waitany_fn = NNTI_ptl_waitany; - available_transports[trans_id].ops.nnti_waitall_fn = NNTI_ptl_waitall; - available_transports[trans_id].ops.nnti_fini_fn = NNTI_ptl_fini; - } -#endif -#if defined(HAVE_TRIOS_INFINIBAND) - if (trans_id == NNTI_TRANSPORT_IB) { - available_transports[trans_id].initialized = 1; - available_transports[trans_id].ops.nnti_init_fn = NNTI_ib_init; - available_transports[trans_id].ops.nnti_get_url_fn = NNTI_ib_get_url; - available_transports[trans_id].ops.nnti_connect_fn = NNTI_ib_connect; - available_transports[trans_id].ops.nnti_disconnect_fn = NNTI_ib_disconnect; - available_transports[trans_id].ops.nnti_alloc_fn = NNTI_ib_alloc; - available_transports[trans_id].ops.nnti_free_fn = NNTI_ib_free; - available_transports[trans_id].ops.nnti_register_memory_fn = NNTI_ib_register_memory; - available_transports[trans_id].ops.nnti_register_segments_fn = NNTI_ib_register_segments; - available_transports[trans_id].ops.nnti_unregister_memory_fn = NNTI_ib_unregister_memory; - available_transports[trans_id].ops.nnti_send_fn = NNTI_ib_send; - available_transports[trans_id].ops.nnti_put_fn = NNTI_ib_put; - available_transports[trans_id].ops.nnti_get_fn = NNTI_ib_get; - available_transports[trans_id].ops.nnti_scatter_fn = NNTI_ib_scatter; - available_transports[trans_id].ops.nnti_gather_fn = NNTI_ib_gather; - available_transports[trans_id].ops.nnti_atomic_set_callback_fn = NNTI_ib_atomic_set_callback; - available_transports[trans_id].ops.nnti_atomic_read_fn = NNTI_ib_atomic_read; - available_transports[trans_id].ops.nnti_atomic_fop_fn = NNTI_ib_atomic_fop; - available_transports[trans_id].ops.nnti_atomic_cswap_fn = NNTI_ib_atomic_cswap; - available_transports[trans_id].ops.nnti_create_work_request_fn = NNTI_ib_create_work_request; - available_transports[trans_id].ops.nnti_clear_work_request_fn = NNTI_ib_clear_work_request; - available_transports[trans_id].ops.nnti_destroy_work_request_fn = NNTI_ib_destroy_work_request; - available_transports[trans_id].ops.nnti_cancel_fn = NNTI_ib_cancel; - available_transports[trans_id].ops.nnti_cancelall_fn = NNTI_ib_cancelall; - available_transports[trans_id].ops.nnti_interrupt_fn = NNTI_ib_interrupt; - available_transports[trans_id].ops.nnti_wait_fn = NNTI_ib_wait; - available_transports[trans_id].ops.nnti_waitany_fn = NNTI_ib_waitany; - available_transports[trans_id].ops.nnti_waitall_fn = NNTI_ib_waitall; - available_transports[trans_id].ops.nnti_fini_fn = NNTI_ib_fini; - } -#endif -#if defined(HAVE_TRIOS_GEMINI) - if (trans_id == NNTI_TRANSPORT_GEMINI) { - available_transports[trans_id].initialized = 1; - available_transports[trans_id].ops.nnti_init_fn = NNTI_gni_init; - available_transports[trans_id].ops.nnti_get_url_fn = NNTI_gni_get_url; - available_transports[trans_id].ops.nnti_connect_fn = NNTI_gni_connect; - available_transports[trans_id].ops.nnti_disconnect_fn = NNTI_gni_disconnect; - available_transports[trans_id].ops.nnti_alloc_fn = NNTI_gni_alloc; - available_transports[trans_id].ops.nnti_free_fn = NNTI_gni_free; - available_transports[trans_id].ops.nnti_register_memory_fn = NNTI_gni_register_memory; - available_transports[trans_id].ops.nnti_register_segments_fn = NNTI_gni_register_segments; - available_transports[trans_id].ops.nnti_unregister_memory_fn = NNTI_gni_unregister_memory; - available_transports[trans_id].ops.nnti_send_fn = NNTI_gni_send; - available_transports[trans_id].ops.nnti_put_fn = NNTI_gni_put; - available_transports[trans_id].ops.nnti_get_fn = NNTI_gni_get; - available_transports[trans_id].ops.nnti_scatter_fn = NNTI_gni_scatter; - available_transports[trans_id].ops.nnti_gather_fn = NNTI_gni_gather; - available_transports[trans_id].ops.nnti_atomic_set_callback_fn = NNTI_gni_atomic_set_callback; - available_transports[trans_id].ops.nnti_atomic_read_fn = NNTI_gni_atomic_read; - available_transports[trans_id].ops.nnti_atomic_fop_fn = NNTI_gni_atomic_fop; - available_transports[trans_id].ops.nnti_atomic_cswap_fn = NNTI_gni_atomic_cswap; - available_transports[trans_id].ops.nnti_create_work_request_fn = NNTI_gni_create_work_request; - available_transports[trans_id].ops.nnti_clear_work_request_fn = NNTI_gni_clear_work_request; - available_transports[trans_id].ops.nnti_destroy_work_request_fn = NNTI_gni_destroy_work_request; - available_transports[trans_id].ops.nnti_cancel_fn = NNTI_gni_cancel; - available_transports[trans_id].ops.nnti_cancelall_fn = NNTI_gni_cancelall; - available_transports[trans_id].ops.nnti_interrupt_fn = NNTI_gni_interrupt; - available_transports[trans_id].ops.nnti_wait_fn = NNTI_gni_wait; - available_transports[trans_id].ops.nnti_waitany_fn = NNTI_gni_waitany; - available_transports[trans_id].ops.nnti_waitall_fn = NNTI_gni_waitall; - available_transports[trans_id].ops.nnti_fini_fn = NNTI_gni_fini; - } -#endif -#if defined(HAVE_TRIOS_BGPDCMF) - if (trans_id == NNTI_TRANSPORT_DCMF) { - available_transports[trans_id].initialized = 1; - available_transports[trans_id].ops.nnti_init_fn = NNTI_bgpdcmf_init; - available_transports[trans_id].ops.nnti_get_url_fn = NNTI_bgpdcmf_get_url; - available_transports[trans_id].ops.nnti_connect_fn = NNTI_bgpdcmf_connect; - available_transports[trans_id].ops.nnti_disconnect_fn = NNTI_bgpdcmf_disconnect; - available_transports[trans_id].ops.nnti_alloc_fn = NNTI_bgpdcmf_alloc; - available_transports[trans_id].ops.nnti_free_fn = NNTI_bgpdcmf_free; - available_transports[trans_id].ops.nnti_register_memory_fn = NNTI_bgpdcmf_register_memory; - available_transports[trans_id].ops.nnti_register_segments_fn = NNTI_bgpdcmf_register_segments; - available_transports[trans_id].ops.nnti_unregister_memory_fn = NNTI_bgpdcmf_unregister_memory; - available_transports[trans_id].ops.nnti_send_fn = NNTI_bgpdcmf_send; - available_transports[trans_id].ops.nnti_put_fn = NNTI_bgpdcmf_put; - available_transports[trans_id].ops.nnti_get_fn = NNTI_bgpdcmf_get; - available_transports[trans_id].ops.nnti_scatter_fn = NNTI_bgpdcmf_scatter; - available_transports[trans_id].ops.nnti_gather_fn = NNTI_bgpdcmf_gather; - available_transports[trans_id].ops.nnti_atomic_set_callback_fn = NNTI_bgpdcmf_atomic_set_callback; - available_transports[trans_id].ops.nnti_atomic_read_fn = NNTI_bgpdcmf_atomic_read; - available_transports[trans_id].ops.nnti_atomic_fop_fn = NNTI_bgpdcmf_atomic_fop; - available_transports[trans_id].ops.nnti_atomic_cswap_fn = NNTI_bgpdcmf_atomic_cswap; - available_transports[trans_id].ops.nnti_create_work_request_fn = NNTI_bgpdcmf_create_work_request; - available_transports[trans_id].ops.nnti_clear_work_request_fn = NNTI_bgpdcmf_clear_work_request; - available_transports[trans_id].ops.nnti_destroy_work_request_fn = NNTI_bgpdcmf_destroy_work_request; - available_transports[trans_id].ops.nnti_cancel_fn = NNTI_bgpdcmf_cancel; - available_transports[trans_id].ops.nnti_cancelall_fn = NNTI_bgpdcmf_cancelall; - available_transports[trans_id].ops.nnti_interrupt_fn = NNTI_bgpdcmf_interrupt; - available_transports[trans_id].ops.nnti_wait_fn = NNTI_bgpdcmf_wait; - available_transports[trans_id].ops.nnti_waitany_fn = NNTI_bgpdcmf_waitany; - available_transports[trans_id].ops.nnti_waitall_fn = NNTI_bgpdcmf_waitall; - available_transports[trans_id].ops.nnti_fini_fn = NNTI_bgpdcmf_fini; - } -#endif -#if defined(HAVE_TRIOS_BGQPAMI) - if (trans_id == NNTI_TRANSPORT_PAMI) { - available_transports[trans_id].initialized = 1; - available_transports[trans_id].ops.nnti_init_fn = NNTI_bgqpami_init; - available_transports[trans_id].ops.nnti_get_url_fn = NNTI_bgqpami_get_url; - available_transports[trans_id].ops.nnti_connect_fn = NNTI_bgqpami_connect; - available_transports[trans_id].ops.nnti_disconnect_fn = NNTI_bgqpami_disconnect; - available_transports[trans_id].ops.nnti_alloc_fn = NNTI_bgqpami_alloc; - available_transports[trans_id].ops.nnti_free_fn = NNTI_bgqpami_free; - available_transports[trans_id].ops.nnti_register_memory_fn = NNTI_bgqpami_register_memory; - available_transports[trans_id].ops.nnti_register_segments_fn = NNTI_bgqpami_register_segments; - available_transports[trans_id].ops.nnti_unregister_memory_fn = NNTI_bgqpami_unregister_memory; - available_transports[trans_id].ops.nnti_send_fn = NNTI_bgqpami_send; - available_transports[trans_id].ops.nnti_put_fn = NNTI_bgqpami_put; - available_transports[trans_id].ops.nnti_get_fn = NNTI_bgqpami_get; - available_transports[trans_id].ops.nnti_scatter_fn = NNTI_bgqpami_scatter; - available_transports[trans_id].ops.nnti_gather_fn = NNTI_bgqpami_gather; - available_transports[trans_id].ops.nnti_atomic_set_callback_fn = NNTI_bgqpami_atomic_set_callback; - available_transports[trans_id].ops.nnti_atomic_read_fn = NNTI_bgqpami_atomic_read; - available_transports[trans_id].ops.nnti_atomic_fop_fn = NNTI_bgqpami_atomic_fop; - available_transports[trans_id].ops.nnti_atomic_cswap_fn = NNTI_bgqpami_atomic_cswap; - available_transports[trans_id].ops.nnti_create_work_request_fn = NNTI_bgqpami_create_work_request; - available_transports[trans_id].ops.nnti_clear_work_request_fn = NNTI_bgqpami_clear_work_request; - available_transports[trans_id].ops.nnti_destroy_work_request_fn = NNTI_bgqpami_destroy_work_request; - available_transports[trans_id].ops.nnti_cancel_fn = NNTI_bgqpami_cancel; - available_transports[trans_id].ops.nnti_cancelall_fn = NNTI_bgqpami_cancelall; - available_transports[trans_id].ops.nnti_interrupt_fn = NNTI_bgqpami_interrupt; - available_transports[trans_id].ops.nnti_wait_fn = NNTI_bgqpami_wait; - available_transports[trans_id].ops.nnti_waitany_fn = NNTI_bgqpami_waitany; - available_transports[trans_id].ops.nnti_waitall_fn = NNTI_bgqpami_waitall; - available_transports[trans_id].ops.nnti_fini_fn = NNTI_bgqpami_fini; - } -#endif -#if defined(HAVE_TRIOS_MPI) - if (trans_id == NNTI_TRANSPORT_MPI) { - available_transports[trans_id].initialized = 1; - available_transports[trans_id].ops.nnti_init_fn = NNTI_mpi_init; - available_transports[trans_id].ops.nnti_get_url_fn = NNTI_mpi_get_url; - available_transports[trans_id].ops.nnti_connect_fn = NNTI_mpi_connect; - available_transports[trans_id].ops.nnti_disconnect_fn = NNTI_mpi_disconnect; - available_transports[trans_id].ops.nnti_alloc_fn = NNTI_mpi_alloc; - available_transports[trans_id].ops.nnti_free_fn = NNTI_mpi_free; - available_transports[trans_id].ops.nnti_register_memory_fn = NNTI_mpi_register_memory; - available_transports[trans_id].ops.nnti_register_segments_fn = NNTI_mpi_register_segments; - available_transports[trans_id].ops.nnti_unregister_memory_fn = NNTI_mpi_unregister_memory; - available_transports[trans_id].ops.nnti_send_fn = NNTI_mpi_send; - available_transports[trans_id].ops.nnti_put_fn = NNTI_mpi_put; - available_transports[trans_id].ops.nnti_get_fn = NNTI_mpi_get; - available_transports[trans_id].ops.nnti_scatter_fn = NNTI_mpi_scatter; - available_transports[trans_id].ops.nnti_gather_fn = NNTI_mpi_gather; - available_transports[trans_id].ops.nnti_atomic_set_callback_fn = NNTI_mpi_atomic_set_callback; - available_transports[trans_id].ops.nnti_atomic_read_fn = NNTI_mpi_atomic_read; - available_transports[trans_id].ops.nnti_atomic_fop_fn = NNTI_mpi_atomic_fop; - available_transports[trans_id].ops.nnti_atomic_cswap_fn = NNTI_mpi_atomic_cswap; - available_transports[trans_id].ops.nnti_create_work_request_fn = NNTI_mpi_create_work_request; - available_transports[trans_id].ops.nnti_clear_work_request_fn = NNTI_mpi_clear_work_request; - available_transports[trans_id].ops.nnti_destroy_work_request_fn = NNTI_mpi_destroy_work_request; - available_transports[trans_id].ops.nnti_cancel_fn = NNTI_mpi_cancel; - available_transports[trans_id].ops.nnti_cancelall_fn = NNTI_mpi_cancelall; - available_transports[trans_id].ops.nnti_interrupt_fn = NNTI_mpi_interrupt; - available_transports[trans_id].ops.nnti_wait_fn = NNTI_mpi_wait; - available_transports[trans_id].ops.nnti_waitany_fn = NNTI_mpi_waitany; - available_transports[trans_id].ops.nnti_waitall_fn = NNTI_mpi_waitall; - available_transports[trans_id].ops.nnti_fini_fn = NNTI_mpi_fini; - } -#endif - - trans_hdl->datatype = NNTI_dt_transport; - trans_hdl->id = trans_id; - - rc = available_transports[trans_id].ops.nnti_init_fn( - trans_id, - my_url, - trans_hdl); - - return(rc); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recognize. - * - * URL format: "transport://address/memory_descriptor" - * - transport - (required) identifies how the URL should parsed - * - address - (required) uniquely identifies a location on the network - * - ex. "ptl://nid:pid/", "ib://ip_addr:port" - * - memory_descriptor - (optional) transport-specific representation of RMA params - * - */ -NNTI_result_t NNTI_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_get_url_fn( - trans_hdl, - url, - maxlen); - } - - return(rc); -} - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * - */ -NNTI_result_t NNTI_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_connect_fn( - trans_hdl, - url, - timeout, - peer_hdl); - } - - peer_hdl->datatype = NNTI_dt_peer; - - return(rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - * - */ -NNTI_result_t NNTI_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_disconnect_fn( - trans_hdl, - peer_hdl); - } - - return(rc); -} - - -/** - * @brief Allocate a block of memory and prepare it for network operations. - * - * Allocate a block of memory in a transport specific way and wrap it in an - * NNTI_buffer_t. The transport may take additional actions to prepare the - * memory for network send/receive. - * - */ -NNTI_result_t NNTI_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_alloc_fn( - trans_hdl, - element_size, - num_elements, - ops, - reg_buf); - } - - reg_buf->datatype = NNTI_dt_buffer; - - return(rc); -} - - -/** - * @brief Cleanup after network operations are complete and release the block - * of memory. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_alloc(). - * - */ -NNTI_result_t NNTI_free ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[reg_buf->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[reg_buf->transport_id].ops.nnti_free_fn( - reg_buf); - } - - return(rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - * - */ -NNTI_result_t NNTI_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - if (ops == NNTI_BOP_RECV_QUEUE) { - log_error(nnti_debug_level, "NNTI_BOP_RECV_QUEUE type require the use of NNTI_alloc()."); - rc=NNTI_EINVAL; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_register_memory_fn( - trans_hdl, - buffer, - element_size, - num_elements, - ops, - reg_buf); - } - } - - reg_buf->datatype = NNTI_dt_buffer; - - return(rc); -} - - -/** - * @brief Prepare a list of memory segments for network operations. - * - * Wrap a list of user allocated memory segments in an NNTI_buffer_t. The - * transport may take additional actions to prepare the memory segments for - * network send/receive. If the memory segments don't meet the transport's - * requirements for memory regions, then errors or poor performance may - * result. - * - */ -NNTI_result_t NNTI_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - if (ops == NNTI_BOP_RECV_QUEUE) { - log_error(nnti_debug_level, "NNTI_BOP_RECV_QUEUE type cannot be segmented."); - rc=NNTI_EINVAL; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_register_segments_fn( - trans_hdl, - segments, - segment_lengths, - num_segments, - ops, - reg_buf); - } - } - - reg_buf->datatype = NNTI_dt_buffer; - - return(rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - * - */ -NNTI_result_t NNTI_unregister_memory ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[reg_buf->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[reg_buf->transport_id].ops.nnti_unregister_memory_fn( - reg_buf); - } - - return(rc); -} - - -/** - * @brief Calculate the number of bytes required to store an encoded NNTI datatype. - * - */ -NNTI_result_t NNTI_dt_sizeof ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt, - uint64_t *packed_len) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - xdrproc_t sizeof_fn; - NNTI_datatype_t *dt=(NNTI_datatype_t*)nnti_dt; - - switch (*dt) { - case NNTI_dt_transport: - sizeof_fn=(xdrproc_t)&xdr_NNTI_transport_t; - break; - case NNTI_dt_peer: - sizeof_fn=(xdrproc_t)&xdr_NNTI_peer_t; - break; - case NNTI_dt_buffer: - sizeof_fn=(xdrproc_t)&xdr_NNTI_buffer_t; - break; - case NNTI_dt_work_request: - sizeof_fn=(xdrproc_t)&xdr_NNTI_work_request_t; - break; - case NNTI_dt_status: - sizeof_fn=(xdrproc_t)&xdr_NNTI_status_t; - break; - } - *packed_len = xdr_sizeof(sizeof_fn, nnti_dt); - *packed_len += sizeof(NNTI_datatype_t); - } - - return(rc); -} - - -/** - * @brief Encode an NNTI datatype into an array of bytes. - * - */ -NNTI_result_t NNTI_dt_pack ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt, - char *packed_buf, - uint64_t packed_buflen) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - XDR encode_xdrs; - xdrproc_t encode_fn; - NNTI_datatype_t *dt=(NNTI_datatype_t*)nnti_dt; - - switch (*dt) { - case NNTI_dt_transport: - encode_fn=(xdrproc_t)&xdr_NNTI_transport_t; - break; - case NNTI_dt_peer: - encode_fn=(xdrproc_t)&xdr_NNTI_peer_t; - break; - case NNTI_dt_buffer: - encode_fn=(xdrproc_t)&xdr_NNTI_buffer_t; - break; - case NNTI_dt_work_request: - encode_fn=(xdrproc_t)&xdr_NNTI_work_request_t; - break; - case NNTI_dt_status: - encode_fn=(xdrproc_t)&xdr_NNTI_status_t; - break; - } - - *(NNTI_datatype_t*)packed_buf = *dt; - - packed_buf += sizeof(NNTI_datatype_t); - packed_buflen -= sizeof(NNTI_datatype_t); - - xdrmem_create( - &encode_xdrs, - packed_buf, - packed_buflen, - XDR_ENCODE); - - if (!encode_fn(&encode_xdrs, nnti_dt)) { - log_fatal(nnti_debug_level,"packing failed"); - return NNTI_EENCODE; - } - } - - return(rc); -} - - -/** - * @brief Decode an array of bytes into an NNTI datatype. - * - */ -NNTI_result_t NNTI_dt_unpack ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt, - char *packed_buf, - uint64_t packed_buflen) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - XDR decode_xdrs; - xdrproc_t decode_fn; - NNTI_datatype_t *dt=(NNTI_datatype_t*)packed_buf; - uint64_t dt_size; - - switch (*dt) { - case NNTI_dt_transport: - decode_fn=(xdrproc_t)&xdr_NNTI_transport_t; - dt_size=sizeof(NNTI_transport_t); - break; - case NNTI_dt_peer: - decode_fn=(xdrproc_t)&xdr_NNTI_peer_t; - dt_size=sizeof(NNTI_peer_t); - break; - case NNTI_dt_buffer: - decode_fn=(xdrproc_t)&xdr_NNTI_buffer_t; - dt_size=sizeof(NNTI_buffer_t); - break; - case NNTI_dt_work_request: - decode_fn=(xdrproc_t)&xdr_NNTI_work_request_t; - dt_size=sizeof(NNTI_work_request_t); - break; - case NNTI_dt_status: - decode_fn=(xdrproc_t)&xdr_NNTI_status_t; - dt_size=sizeof(NNTI_status_t); - break; - } - - packed_buf += sizeof(NNTI_datatype_t); - packed_buflen -= sizeof(NNTI_datatype_t); - - memset(nnti_dt, 0, dt_size); - xdrmem_create( - &decode_xdrs, - packed_buf, - packed_buflen, - XDR_DECODE); - if (!decode_fn(&decode_xdrs, nnti_dt)) { - log_fatal(nnti_debug_level,"unpacking failed"); - return NNTI_EDECODE; - } - } - - return(rc); -} - - -/** - * @brief Free a variable size NNTI datatype that was unpacked with NNTI_dt_unpack(). - * - */ -NNTI_result_t NNTI_dt_free ( - const NNTI_transport_t *trans_hdl, - void *nnti_dt) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - xdrproc_t free_fn; - NNTI_datatype_t *dt=(NNTI_datatype_t*)nnti_dt; - - switch (*dt) { - case NNTI_dt_transport: - free_fn=(xdrproc_t)&xdr_NNTI_transport_t; - break; - case NNTI_dt_peer: - free_fn=(xdrproc_t)&xdr_NNTI_peer_t; - break; - case NNTI_dt_buffer: - free_fn=(xdrproc_t)&xdr_NNTI_buffer_t; - break; - case NNTI_dt_work_request: - free_fn=(xdrproc_t)&xdr_NNTI_work_request_t; - break; - case NNTI_dt_status: - free_fn=(xdrproc_t)&xdr_NNTI_status_t; - break; - } - - xdr_free(free_fn, nnti_dt); - } - - return(rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - * - */ -NNTI_result_t NNTI_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[msg_hdl->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[msg_hdl->transport_id].ops.nnti_send_fn( - peer_hdl, - msg_hdl, - dest_hdl, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[src_buffer_hdl->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[src_buffer_hdl->transport_id].ops.nnti_put_fn( - src_buffer_hdl, - src_offset, - src_length, - dest_buffer_hdl, - dest_offset, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[dest_buffer_hdl->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[dest_buffer_hdl->transport_id].ops.nnti_get_fn( - src_buffer_hdl, - src_offset, - src_length, - dest_buffer_hdl, - dest_offset, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Transfer data to a peer. - * - */ -NNTI_result_t NNTI_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[src_buffer_hdl->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[src_buffer_hdl->transport_id].ops.nnti_scatter_fn( - src_buffer_hdl, - src_length, - dest_buffer_list, - dest_count, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Transfer data from a peer. - * - */ -NNTI_result_t NNTI_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[dest_buffer_hdl->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[dest_buffer_hdl->transport_id].ops.nnti_gather_fn( - src_buffer_list, - src_length, - src_count, - dest_buffer_hdl, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -NNTI_result_t NNTI_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_atomic_set_callback_fn( - trans_hdl, - local_atomic, - cbfunc, - context); - } - - return(rc); -} - - -NNTI_result_t NNTI_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_atomic_read_fn( - trans_hdl, - local_atomic, - value); - } - - return(rc); -} - - -NNTI_result_t NNTI_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_atomic_fop_fn( - trans_hdl, - peer_hdl, - target_atomic, - result_atomic, - operand, - op, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -NNTI_result_t NNTI_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_atomic_cswap_fn( - trans_hdl, - peer_hdl, - target_atomic, - result_atomic, - compare_operand, - swap_operand, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Create a receive work request that can be used to wait for buffer - * operations to complete. - * - */ -NNTI_result_t NNTI_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[reg_buf->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[reg_buf->transport_id].ops.nnti_create_work_request_fn( - reg_buf, - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Disassociates a receive work request from a previous receive - * and prepares it for reuse. - * - */ -NNTI_result_t NNTI_clear_work_request ( - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[wr->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[wr->transport_id].ops.nnti_clear_work_request_fn( - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Disassociates a receive work request from reg_buf. - * - */ -NNTI_result_t NNTI_destroy_work_request ( - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[wr->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[wr->transport_id].ops.nnti_destroy_work_request_fn( - wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Attempts to cancel an NNTI opertion. - * - */ -NNTI_result_t NNTI_cancel ( - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[wr->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[wr->transport_id].ops.nnti_cancel_fn(wr); - } - - wr->datatype = NNTI_dt_work_request; - - return(rc); -} - - -/** - * @brief Attempts to cancel a list of NNTI opertions. - * - */ -NNTI_result_t NNTI_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - NNTI_result_t rc=NNTI_OK; - NNTI_transport_id_t id=NNTI_TRANSPORT_NULL; - uint32_t i=0; - - for (i=0;itransport_id; - } - } - - if (id == NNTI_TRANSPORT_NULL) { - rc=NNTI_EINVAL; - } else { - if (available_transports[id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[id].ops.nnti_cancelall_fn( - wr_list, - wr_count); - } - } - - for (i=0;idatatype = NNTI_dt_work_request; - } - } - - return(rc); -} - - -/** - * @brief Interrupts NNTI_wait*() - * - */ -NNTI_result_t NNTI_interrupt ( - const NNTI_transport_t *trans_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_interrupt_fn( - trans_hdl); - } - - return(rc); -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t NNTI_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[wr->transport_id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[wr->transport_id].ops.nnti_wait_fn( - wr, - timeout, - status); - } - - status->datatype = NNTI_dt_status; - - return(rc); -} - - -/** - * @brief Wait for remote_op on any buffer in buf_list to complete. - * - * Wait for remote_op on any buffer in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the receive queue and RDMA buffers in the same call. Will probably be fixed in the future. - * - */ -NNTI_result_t NNTI_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status) -{ - NNTI_result_t rc=NNTI_OK; - NNTI_transport_id_t id=NNTI_TRANSPORT_NULL; - uint32_t i=0; - - for (i=0;itransport_id; - } - } - - if (id == NNTI_TRANSPORT_NULL) { - rc=NNTI_EINVAL; - } else { - if (available_transports[id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[id].ops.nnti_waitany_fn( - wr_list, - wr_count, - timeout, - which, - status); - } - } - - status->datatype = NNTI_dt_status; - - return(rc); -} - - -/** - * @brief Wait for remote_op on all buffers in buf_list to complete. - * - * Wait for remote_op on all buffers in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the receive queue and RDMA buffers in the same call. Will probably be fixed in the future. - * - */ -NNTI_result_t NNTI_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status) -{ - NNTI_result_t rc=NNTI_OK; - NNTI_transport_id_t id=NNTI_TRANSPORT_NULL; - uint32_t i=0; - - for (i=0;itransport_id; - } - } - - if (id == NNTI_TRANSPORT_NULL) { - rc=NNTI_EINVAL; - } else { - if (available_transports[id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[id].ops.nnti_waitall_fn( - wr_list, - wr_count, - timeout, - status); - } - } - - for (i=0;idatatype = NNTI_dt_status; - } - } - - return(rc); -} - - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t NNTI_fini ( - const NNTI_transport_t *trans_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - if (available_transports[trans_hdl->id].initialized==0) { - rc=NNTI_ENOTINIT; - } else { - rc = available_transports[trans_hdl->id].ops.nnti_fini_fn( - trans_hdl); - memset(&available_transports[trans_hdl->id], 0, sizeof(NNTI_internal_transport_t)); - } - - return(rc); -} diff --git a/packages/trios/libraries/nessie/nnti/nnti_dcmf.cpp b/packages/trios/libraries/nessie/nnti/nnti_dcmf.cpp deleted file mode 100644 index 8523c583945d..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_dcmf.cpp +++ /dev/null @@ -1,3332 +0,0 @@ -/* - * nnti_dcmf.c - * - * Created on: April 19, 2012 - */ - -#include "Trios_config.h" -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include "nnti_dcmf.h" -#include "nnti_utils.h" -#include "nnti_internal.h" - -#define TRACE_ERR(x) /* fprintf x */ - -/* GLOBAL VAR */ - -DCMF_Protocol_t send_prot; -DCMF_Protocol_t control0_prot; -DCMF_Control_t info; -DCMF_Protocol_t req_index_prot; - -DCMF_Send_Configuration_t conf; /* Register the send protocol for intial conn */ - -#define BUFSIZE 2048 -#define ITERATIONS 1000 -#define MAX_CONNECTION 1024 -size_t my_rank; -volatile unsigned _recv_active; -volatile unsigned send_active; -volatile unsigned _recv_iteration; -volatile unsigned confirm_conn; -DCMF_Request_t _recv_request[ITERATIONS]; -char _recv_buffer[BUFSIZE] __attribute__ ((__aligned__ (16))); -volatile static int _is_server; - -/*** End of GLOBAL VAR ***/ - - -/** - * These states are used to signal events between the completion handler - * and the main client or server thread. - * - * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, - * and RDMA_WRITE_COMPLETE for each ping. - */ -typedef enum -{ - IDLE = 1, - CONNECT_REQUEST, - CONNECTED, - DISCONNECTED, - ERROR -} bgpdcmf_connection_state; - -typedef enum -{ - SERVER_CONNECTION, - CLIENT_CONNECTION -} bgpdcmf_connection_type; - -typedef enum -{ - REQUEST_BUFFER, - RESULT_BUFFER, - RECEIVE_BUFFER, - SEND_BUFFER, - GET_SRC_BUFFER, - GET_DST_BUFFER, - PUT_SRC_BUFFER, - PUT_DST_BUFFER, - RDMA_TARGET_BUFFER, - UNKNOWN_BUFFER -} bgpdcmf_buffer_type; - - -#define DCMF_OP_PUT_INITIATOR 1 -#define DCMF_OP_GET_INITIATOR 2 -#define DCMF_OP_PUT_TARGET 3 -#define DCMF_OP_GET_TARGET 4 -#define DCMF_OP_SEND_REQ 5 -#define DCMF_OP_NEW_REQUEST 6 -#define DCMF_OP_RESULT 7 -#define DCMF_OP_RECEIVE 8 -#define DCMF_OP_REGISTER_RDMA 9 - -typedef enum -{ - SEND_COMPLETE, - RECV_COMPLETE, - RDMA_WRITE_INIT, - RDMA_TARGET_INIT, - RDMA_READ_INIT, - RDMA_COMPLETE -} bgpdcmf_op_state_t; - - -typedef struct -{ - - size_t req_rank; - DCMF_Memregion_t response_hdl; -} nnti_bgpdcmf_fetch_index_req ALIGN_QUADWORD; - -typedef struct -{ - uint32_t req_received; - int local_send_complete; - int local_get_complete; - int remote_put_complete; - int op_complete; - uint32_t req_rank; - uint64_t byte_len; - uint64_t byte_offset; - uint64_t src_offset; - uint64_t dest_offset; - uint16_t op; -} nnti_bgpdcmf_work_completion ALIGN_QUADWORD; - -/** - * attrs to send to the client - */ -typedef struct -{ - uint64_t req_index_addr; /* address of the request index var on the server */ - DCMF_Memregion_t req_index_mem_hdl; - - uint64_t req_buffer_addr; /* address of the request buffer */ - uint64_t req_size; /* the maximum size of a request */ - uint64_t req_count; /* the number of requests that will fit in the queue */ - DCMF_Memregion_t req_mem_hdl; - uint64_t wc_buffer_addr; /* address of the work completion array */ - DCMF_Memregion_t wc_mem_hdl; - uint64_t unblock_buffer_addr; /* address of the fetch_req_array */ - DCMF_Memregion_t unblock_mem_hdl; -} nnti_bgpdcmf_server_queue_attrs; - -/** - * attrs to send to the server - */ -typedef struct -{ - uint64_t req_index; /* this buffer contains the current index of the request queue */ - uint64_t req_index_addr; /* address of the index buffer */ - DCMF_Memregion_t req_index_mem_hdl; /* mem hdl to send to the server */ - DCMF_Memregion_t fetch_index_hdl; -} nnti_bgpdcmf_client_queue_attrs; - -typedef union -{ - nnti_bgpdcmf_client_queue_attrs client; - nnti_bgpdcmf_server_queue_attrs server; -} nnti_bgpdcmf_queue_remote_attrs; - -typedef struct -{ - uint32_t peer_rank; - nnti_bgpdcmf_client_queue_attrs queue_local_attrs; - nnti_bgpdcmf_queue_remote_attrs queue_remote_attrs; - bgpdcmf_connection_state state; - bgpdcmf_connection_type connection_type; -} bgpdcmf_connection; - - -typedef struct -{ - uint8_t is_initiator; - - const NNTI_buffer_t *reg_buf; - nnti_bgpdcmf_work_completion wc; - uint64_t wc_dest_addr; - DCMF_Memregion_t wc_dest_mem_hdl; - DCMF_Memregion_t wc_mem_hdl; - uint8_t last_op; - bgpdcmf_op_state_t op_state; - uint64_t is_last_op_complete; - uint32_t peer_rank; -} dcmf_work_request; - - -typedef - std::deque < -dcmf_work_request * > - wr_queue_t; -typedef - std::deque < -dcmf_work_request * >::iterator - wr_queue_iter_t; - -typedef struct -{ - bgpdcmf_buffer_type - type; - DCMF_Memregion_t - mem_hdl; /* actual data memory handle */ - wr_queue_t - wr_queue; - nthread_lock_t - wr_queue_lock; - uint32_t - ref_count; -} bgpdcmf_memory_handle; - -typedef struct -{ - NNTI_buffer_t * - reg_buf; - - uint64_t - last_index_before_reset; - uint64_t - req_index; /* index of the next available slot in the request queue */ - uint64_t - req_index_addr; /* address of the request index var on the server */ - DCMF_Memregion_t - req_index_mem_hdl; - - char * - req_buffer; /* pointer to the head of the request buffer */ - uint64_t - req_size; /* the maximum size of a request */ - uint64_t - req_count; /* the number of requests that will fit in the queue */ - uint64_t - req_buffer_size; /* the size of the request buffer in bytes (req_size*req_count) */ - DCMF_Memregion_t - req_mem_hdl; - - nnti_bgpdcmf_work_completion * - wc_buffer; /* pointer to work completion array */ - uint64_t - wc_buffer_size; /* the size of the work completion buffer in bytes */ - DCMF_Memregion_t - wc_mem_hdl; - nnti_bgpdcmf_fetch_index_req * - unblock_buffer; /* pointer to individual fetch index array */ - DCMF_Memregion_t - unblock_mem_hdl; - uint64_t - req_processed_reset_limit; - uint64_t - req_processed; - uint64_t - total_req_processed; -} bgpdcmf_request_queue_handle; - -typedef struct -{ - uint32_t - myrank; - uint32_t - remote_rank; - int - mypid; - size_t conn_req[1024]; - bgpdcmf_request_queue_handle - req_queue; -} bgpdcmf_transport_global; - -static nthread_lock_t - nnti_bgpdcmf_lock; -static nthread_lock_t - nnti_index_lock; -static int -start_connection_listener_thread (void); -static int -start_index_thread (void); -static NNTI_result_t -register_memory (bgpdcmf_memory_handle * hdl, - void *buf, uint64_t len, const NNTI_buf_ops_t remote_op); -static int -unregister_memory (bgpdcmf_memory_handle * hdl); -static NNTI_result_t -process_event (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - dcmf_work_request * wr, const int timeout); -/* NEW SHYAMALI */ - -static NNTI_result_t -repost_recv_work_request (NNTI_buffer_t * reg_buf, dcmf_work_request * wr); - -static dcmf_work_request * -first_incomplete_wr (bgpdcmf_memory_handle * _hdl); -static int8_t -is_buf_op_complete (const NNTI_buffer_t * reg_buf); -static int8_t -is_wr_complete (dcmf_work_request * wr); -static int8_t -is_any_buf_op_complete (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, uint32_t * which); -static int8_t -is_all_buf_ops_complete (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count); -static void -create_status (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - int nnti_rc, NNTI_status_t * status); - -/* NEW */ -static void -create_peer (NNTI_peer_t * peer, int rank); -static NNTI_result_t -inject_full (int rank, const void *buf, size_t num); -static NNTI_result_t -init_connection (bgpdcmf_connection ** conn, int rank, const int is_server); -static void -close_connection (bgpdcmf_connection * c); - -static NNTI_result_t -insert_conn_rank (const uint32_t rank, bgpdcmf_connection * conn); -static bgpdcmf_connection * -get_conn_rank (const uint32_t rank); -static bgpdcmf_connection * -del_conn_rank (const uint32_t rank); - -static int -server_req_queue_init (bgpdcmf_request_queue_handle * q, - char *buffer, uint64_t req_size, uint64_t req_count); -static int -server_req_queue_destroy (bgpdcmf_request_queue_handle * q); - -static int -client_req_queue_init (bgpdcmf_connection * c); -static int -client_req_queue_destroy (bgpdcmf_connection * c); - -static int -reset_req_index (bgpdcmf_request_queue_handle * req_queue_attrs); - -static int -fetch_server_req_buffer_offset (nnti_bgpdcmf_client_queue_attrs * - local_req_queue_attrs, - nnti_bgpdcmf_server_queue_attrs * - remote_req_queue_attrs, - uint64_t addend, uint64_t * offset, int rank); -static int -send_req (nnti_bgpdcmf_client_queue_attrs * local_req_queue_attrs, - nnti_bgpdcmf_server_queue_attrs * remote_req_queue_attrs, - uint64_t offset, const NNTI_buffer_t * reg_buf, - int rank, dcmf_work_request * wr); - -static NNTI_result_t -send_req_wc (nnti_bgpdcmf_client_queue_attrs * local_req_queue_attrs, - nnti_bgpdcmf_server_queue_attrs * remote_req_queue_attrs, - uint64_t offset, - const NNTI_buffer_t * reg_buf, dcmf_work_request * wr, int rank); - -static int -request_send (nnti_bgpdcmf_client_queue_attrs * client_q, - nnti_bgpdcmf_server_queue_attrs * server_q, - const NNTI_buffer_t * reg_buf, int rank); - -/* NEW */ -static NNTI_result_t -register_wc (dcmf_work_request * wr); -static NNTI_result_t -insert_buf_bufhash (NNTI_buffer_t * buf); -static NNTI_buffer_t * -get_buf_bufhash (const uint32_t bufhash); -static NNTI_buffer_t * -del_buf_bufhash (NNTI_buffer_t * buf); -static void -print_bufhash_map (void); - -static NNTI_result_t -insert_wr_wrhash (dcmf_work_request *); -static dcmf_work_request * -get_wr_wrhash (const uint32_t bufhash); -static dcmf_work_request * -del_wr_wrhash (dcmf_work_request *); -static void -print_wrhash_map (void); -static void -send_rdma_wc (dcmf_work_request * wr, - const NNTI_buffer_t * local_buf, - const NNTI_buffer_t * remote_buf); - - - -/* Thomas Wang's 64 bit to 32 bit Hash Function (http://www.concentric.net/~ttwang/tech/inthash.htm) */ -static uint32_t -hash6432shift (uint64_t key) -{ - key = (~key) + (key << 18); // key = (key << 18) - key - 1; - key = key ^ (key >> 31); - key = key * 21; // key = (key + (key << 2)) + (key << 4); - key = key ^ (key >> 11); - key = key + (key << 6); - key = key ^ (key >> 22); - return (uint32_t) key; -} - -/* - * We need a couple of maps to keep track of connections. Servers need to find - * connections by QP number when requests arrive. Clients need to find connections - * by peer address and port. Setup those maps here. - */ -static - std::map < int, -bgpdcmf_connection * >connections_by_rank; -typedef - std::map < int, -bgpdcmf_connection * >::iterator - conn_by_rank_iter_t; -typedef - std::pair < int, -bgpdcmf_connection * > - conn_by_rank_t; -static nthread_lock_t - nnti_conn_peer_lock; - -static int8_t -is_wr_queue_empty (const NNTI_buffer_t * buf); - -static - std::map < -uint32_t, NNTI_buffer_t * >buffers_by_bufhash; -typedef - std::map < - uint32_t, -NNTI_buffer_t * >::iterator - buf_by_bufhash_iter_t; -typedef - std::pair < - uint32_t, -NNTI_buffer_t * > - buf_by_bufhash_t; -static nthread_lock_t - nnti_buf_bufhash_lock; - -static - std::map < - uint32_t, -dcmf_work_request * > - wr_by_wrhash; -typedef - std::map < - uint32_t, -dcmf_work_request * >::iterator - wr_by_wrhash_iter_t; -typedef - std::pair < - uint32_t, -dcmf_work_request * > - wr_by_wrhash_t; -static nthread_lock_t - nnti_wr_wrhash_lock; - - - -/* NEW */ - -static bgpdcmf_transport_global - transport_global_data; -static const int - MIN_TIMEOUT = 100000000; /* in milliseconds */ -static log_level - nnti_event_debug_level; -static nnti_bgpdcmf_fetch_index_req - client_fetch_req; - -/************************** Callback functions for DCMF_Send / Recv protocol ************/ -/* Low overhead timing function */ - -typedef unsigned long long - ticks; - -static __inline__ ticks -getticks (void) -{ - unsigned int - tbl, - tbu0, - tbu1; - - do - { - __asm__ - __volatile__ ("mftbu %0":"=r" (tbu0)); - __asm__ - __volatile__ ("mftb %0":"=r" (tbl)); - __asm__ - __volatile__ ("mftbu %0":"=r" (tbu1)); - } - while (tbu0 != tbu1); - - return (((unsigned long long) tbu0) << 32) | tbl; -} - - -void * -allocateMemory (size_t size) -{ - void * - ptr; - int - rc; - - rc = posix_memalign (&ptr, 32, size); - if (rc) - return NULL; - - return ptr; -} - - -static void -decrement (void *clientdata, DCMF_Error_t * err) -{ - unsigned * - _clientdata = (unsigned *) clientdata; - --*_clientdata; - TRACE_ERR ((stderr, "(%zd) decrement() clientdata = %p, %d => %d\n", - my_rank, _clientdata, *_clientdata, *_clientdata - 1)); -} - - -void control_recv (void * clientdata, - const DCMF_Control_t * info, - size_t peer) -{ - DCMF_Control_t tmp; - memcpy((void *)&tmp, (const void *)info, sizeof(DCMF_Control_t)); - (*((unsigned *)clientdata))--; - //log_debug(nnti_debug_level, "Received a confirmation on connection\n"); -} - -/* --------------------------------------------------------------- */ - -static void -cb_recv_new_short (void *clientdata, - const DCQuad * msginfo, - unsigned count, size_t peer, const char *src, size_t bytes) -{ - unsigned * - _clientdata = (unsigned *) clientdata; - NNTI_result_t - rc = NNTI_OK; - - bgpdcmf_connection * - conn = NULL; - - memcpy (&_recv_buffer, src, bytes); - nthread_lock (&nnti_bgpdcmf_lock); - if (strncmp (_recv_buffer, "INIT", 4) == 0) - { - conn = get_conn_rank(peer); - if (conn == NULL) { - transport_global_data.conn_req[peer] = peer; - } - } - else if(_is_server == 1){ - struct - { - nnti_bgpdcmf_client_queue_attrs - client_attrs; - } ca_in; - memset (&ca_in, 0, sizeof (ca_in)); - memcpy(&ca_in, src, sizeof (ca_in)); - conn = get_conn_rank(peer); - if (conn != NULL) - conn->queue_remote_attrs.client = ca_in.client_attrs; - DCMF_CriticalSection_enter (0); - DCMF_Control (&control0_prot, DCMF_MATCH_CONSISTENCY, peer, &info); - DCMF_CriticalSection_exit (0); - // log_debug(nnti_debug_level, "(%zd) cb_recv_new_short() recv full client attr bytes %d : clientdata = %p, %d => %d\n", my_rank, bytes, _clientdata, *_clientdata, *_clientdata-1); - } - --*_clientdata; - nthread_unlock (&nnti_bgpdcmf_lock); -} - -/* -------------------------------------------------------------- */ - -static DCMF_Request_t * -cb_recv_new (void *clientdata, - const DCQuad * msginfo, - unsigned count, - size_t senderID, - size_t sndlen, - size_t * rcvlen, char **rcvbuf, DCMF_Callback_t * cb_info) -{ - TRACE_ERR ((stderr, "(%zd) cb_recv_new() clientdata = %p, sndlen = %zd\n", - my_rank, clientdata, sndlen)); - *rcvbuf = (char *) _recv_buffer; - *rcvlen = sndlen > 0 ? sndlen : 1; - cb_info->function = decrement; - cb_info->clientdata = clientdata; - - // log_debug(nnti_debug_level, "cb_recv_new long case\n"); - return &_recv_request[_recv_iteration++]; -} - - - -void -send_once (char *buffer, size_t sndlen, size_t targetrank, - DCMF_Consistency consistency, DCMF_Protocol_t * protocol) -{ - DCQuad - msginfo; - DCMF_Request_t - request; - - send_active = 1; - DCMF_Callback_t - cb_info = { decrement, (void *) &send_active }; - nthread_lock (&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - DCMF_Send (protocol, - &request, - cb_info, consistency, targetrank, sndlen, buffer, &msginfo, 1); - - while (send_active) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - nthread_unlock (&nnti_bgpdcmf_lock); - //log_debug(nnti_debug_level, "Send once completed %d\n", transport_global_data.myrank); - send_active = 1; -} - -int -dump_memregion (DCMF_Memregion_t * memreg) -{ - - fprintf (stderr, "memregion=0x%08x %08x %08x %08x \n", - *((unsigned *) (((char *) (memreg)) + 0)), - *((unsigned *) (((char *) (memreg)) + 4)), - *((unsigned *) (((char *) (memreg)) + 8)), - *((unsigned *) (((char *) (memreg)) + 12))); - return 0; -} - - -/* ==================== */ - - - -/** - * @brief Initialize NNTI to use a specific transport. - * - */ -NNTI_result_t -NNTI_bgpdcmf_init (const NNTI_transport_id_t trans_id, - const char *my_url, NNTI_transport_t * trans_hdl) -{ - static int - initialized = 0; - - NNTI_result_t - rc = NNTI_OK; - int i; - -#ifndef HAS_MPI - DCMF_Messager_initialize (); -#endif - - my_rank = DCMF_Messager_rank (); - - assert (trans_hdl); - - initialized = 0; - _recv_active = 1; - send_active = 1; - confirm_conn = 1; - _recv_iteration = 0; - - if (!initialized) - { - conf.protocol = DCMF_EAGER_SEND_PROTOCOL; - conf.network = DCMF_TORUS_NETWORK; - conf.cb_recv_short = cb_recv_new_short; - conf.cb_recv_short_clientdata = (void *) &_recv_active; - conf.cb_recv = cb_recv_new; - conf.cb_recv_clientdata = (void *) &_recv_active; - DCMF_Result - result = DCMF_Send_register (&send_prot, &conf); - if (result != DCMF_SUCCESS) - { - fprintf (stderr, "DCMF_SEND register failed in _init\n"); - } - - /* Register DCMF control */ - DCMF_Control_Configuration_t c0_conf; - c0_conf.protocol = DCMF_DEFAULT_CONTROL_PROTOCOL; - c0_conf.network = DCMF_TORUS_NETWORK; - c0_conf.cb_recv = control_recv; - c0_conf.cb_recv_clientdata = (void *)&confirm_conn; - result = DCMF_Control_register(&control0_prot, &c0_conf); - - if (result != DCMF_SUCCESS) - { - fprintf (stderr, "DCMF_Control register failed register failed in _init\n"); - } - - memset (&transport_global_data, 0, sizeof (bgpdcmf_transport_global)); - - nthread_lock_init (&nnti_bgpdcmf_lock); - nthread_lock_init (&nnti_index_lock); - nthread_lock_init (&nnti_conn_peer_lock); - nthread_lock_init (&nnti_wr_wrhash_lock); - nthread_lock_init (&nnti_buf_bufhash_lock); - - //log_debug (nnti_debug_level, "my_url=%s", my_url); - - - //log_debug (nnti_debug_level, "initializing Blue Gene DMA DCMF device"); - - transport_global_data.mypid = getpid (); - transport_global_data.myrank = my_rank; - for (i = 0; i< 1024; ++i) - { - transport_global_data.conn_req[i] = -1; - } - - create_peer (&trans_hdl->me, transport_global_data.myrank); - - initialized = TRUE; - } - - //log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recobgpdcmfze. - * - */ -NNTI_result_t -NNTI_bgpdcmf_get_url (const NNTI_transport_t * trans_hdl, - char *url, const uint64_t maxlen) -{ - NNTI_result_t - rc = NNTI_OK; - - assert (trans_hdl); - assert (url); - assert (maxlen > 0); - - //log_debug (nnti_debug_level, "enter"); - - strncpy (url, trans_hdl->me.url, maxlen); - url[maxlen - 1] = '\0'; - - //log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * - * - * Connectionless transport: parse and populate - * Connected transport: parse, connection and populate - * - */ -NNTI_result_t -NNTI_bgpdcmf_connect (const NNTI_transport_t * trans_hdl, - const char *url, - const int timeout, NNTI_peer_t * peer_hdl) -{ - NNTI_result_t - rc = NNTI_OK; - char * - sendbuf; - - char - transport[NNTI_URL_LEN]; - char - address[NNTI_URL_LEN]; - char * - tmp; - char * - temp; - int - pset_rank; - size_t - sndlen; - - bgpdcmf_connection * - conn = NULL; - - - assert (trans_hdl); - assert (peer_hdl); - - //log_debug (nnti_debug_level, "enter URL = %s myrank = %d", url, transport_global_data.myrank); - - if (url != NULL) - { - if ((rc = - nnti_url_get_transport (url, transport, NNTI_URL_LEN)) != NNTI_OK) - { - return (rc); - } - if (0 != strcmp (transport, "dcmf")) - { - /* the peer described by 'url' is not a BGP Torus peer */ - return (NNTI_EINVAL); - } - - if ((rc = nnti_url_get_address (url, address, NNTI_URL_LEN)) != NNTI_OK) - { - return (rc); - } - pset_rank = strtol (address , NULL, 10); - transport_global_data.remote_rank = pset_rank; /* This is server rank */ - //log_debug (nnti_debug_level, "connect client to %d ", pset_rank); - } - else - { - /* */ - rc = NNTI_EINVAL; - goto cleanup; - } - sendbuf = (char *) allocateMemory (256); /* send INIT_CONNECTION wth rank as SW_ARG */ - memset (sendbuf, 0x00, sizeof (sendbuf)); - sprintf (sendbuf, "INIT_CONNECTION:%d:", transport_global_data.myrank); - sndlen = 32; - send_once (sendbuf, sndlen, transport_global_data.remote_rank, - DCMF_SEQUENTIAL_CONSISTENCY, &send_prot); - rc = init_connection (&conn, pset_rank, 0); - if (conn == NULL) - { - fprintf (stderr, "bgpdcmf connect error\n"); - rc = NNTI_EIO; - goto cleanup; - } - create_peer (peer_hdl, pset_rank); - insert_conn_rank (pset_rank, conn); -cleanup: - free (sendbuf); - //log_debug (nnti_debug_level, "exit"); - return (rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - */ -NNTI_result_t -NNTI_bgpdcmf_disconnect (const NNTI_transport_t * trans_hdl, - NNTI_peer_t * peer_hdl) -{ - NNTI_result_t - rc = NNTI_OK; - - assert (trans_hdl); - assert (peer_hdl); - - //log_debug (nnti_debug_level, "enter"); - bgpdcmf_connection * - conn = - get_conn_rank (peer_hdl->peer.NNTI_remote_process_t_u.bgpdcmf.pset_rank); - close_connection (conn); - del_conn_rank (peer_hdl->peer.NNTI_remote_process_t_u.bgpdcmf.pset_rank); - - free (peer_hdl->url); - - //log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - */ -NNTI_result_t -NNTI_bgpdcmf_register_memory (const NNTI_transport_t * trans_hdl, - char *buffer, - const uint64_t size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - const NNTI_peer_t * peer, - NNTI_buffer_t * reg_buf) -{ - NNTI_result_t - rc = NNTI_OK; - - dcmf_work_request * - wr = NULL; - NNTI_buffer_t * - old_buf = NULL; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - - - assert (trans_hdl); - assert (buffer); - assert (size > 0); - assert (ops > 0); - assert (reg_buf); - - log_debug (nnti_debug_level, "register memory enter buffer=%p", reg_buf); - - old_buf = get_buf_bufhash (hash6432shift ((uint64_t) buffer)); - if (old_buf == NULL) - { - bgpdcmf_mem_hdl = new bgpdcmf_memory_handle (); - bgpdcmf_mem_hdl->ref_count = 1; - } - else - { - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) old_buf->transport_private; - bgpdcmf_mem_hdl->ref_count++; - } - - assert (bgpdcmf_mem_hdl); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size = size; - reg_buf->payload = (uint64_t) buffer; - reg_buf->transport_private = (uint64_t) bgpdcmf_mem_hdl; - - if (peer != NULL) - { - reg_buf->buffer_owner = *peer; - } - -/* - log_debug (nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); -*/ - reg_buf->buffer_addr.transport_id = NNTI_TRANSPORT_DCMF; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.size = size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.buf = (uint64_t) buffer; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.owner_rank = - DCMF_Messager_rank (); - if (bgpdcmf_mem_hdl->ref_count == 1) - { - if (ops == NNTI_RECV_QUEUE) - { - bgpdcmf_request_queue_handle * - q_hdl = &transport_global_data.req_queue; - - /* - * This is a receive-only buffer. This buffer is divisible by - * NNTI_REQUEST_BUFFER_SIZE. This buffer can hold more than - * one short request. Assume this buffer is a request queue. - */ - bgpdcmf_mem_hdl->type = REQUEST_BUFFER; - memset (q_hdl, 0, sizeof (bgpdcmf_request_queue_handle)); - q_hdl->reg_buf = reg_buf; /* req buffer can be accessed from global req_queue */ - transport_global_data.req_queue.reg_buf = reg_buf; - //log_debug(nnti_debug_level, "register req buffer for server"); - - server_req_queue_init (q_hdl, buffer, size, num_elements); - - reg_buf->payload_size = q_hdl->req_size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.type = - NNTI_DCMF_REQUEST_BUFFER; - - } - else if (ops == NNTI_RECV_DST) - { - if (size == NNTI_RESULT_BUFFER_SIZE) - { - /* - * This is a receive-only buffer. This buffer can hold exactly - * one short result. Assume this buffer is a result queue. - */ - bgpdcmf_mem_hdl->type = RESULT_BUFFER; - - rc = - register_memory (bgpdcmf_mem_hdl, buffer, - NNTI_RESULT_BUFFER_SIZE, ops); - - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.type = - NNTI_DCMF_RECEIVE_DST; - } - else - { - /* - * This is a receive-only buffer. This buffer doesn't look - * like a request buffer or a result buffer. I don't know - * what it is. Assume it is a regular data buffer. - */ - bgpdcmf_mem_hdl->type = RECEIVE_BUFFER; - - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.type = - NNTI_DCMF_RECEIVE_DST; - - } - - } - else if (ops == NNTI_SEND_SRC) - { - bgpdcmf_mem_hdl->type = SEND_BUFFER; - - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - if (rc != NNTI_OK) - { - fprintf (stderr, - "failed registering short request in register memory\n"); - } - - } - else if (ops == NNTI_GET_DST) - { - bgpdcmf_mem_hdl->type = GET_DST_BUFFER; - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - - } - else if (ops == NNTI_GET_SRC) - { - bgpdcmf_mem_hdl->type = GET_SRC_BUFFER; - - - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - - } - else if (ops == NNTI_PUT_SRC) - { - bgpdcmf_mem_hdl->type = PUT_SRC_BUFFER; - - - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - } - else if (ops == NNTI_PUT_DST) - { - bgpdcmf_mem_hdl->type = PUT_DST_BUFFER; - - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - - } - else if (ops == (NNTI_GET_SRC | NNTI_PUT_DST)) - { - bgpdcmf_mem_hdl->type = RDMA_TARGET_BUFFER; - - rc = register_memory (bgpdcmf_mem_hdl, buffer, size, ops); - } - else - { - bgpdcmf_mem_hdl->type = UNKNOWN_BUFFER; - } - } - wr = (dcmf_work_request *) calloc (1, sizeof (dcmf_work_request)); - wr->reg_buf = reg_buf; - wr->last_op = DCMF_OP_REGISTER_RDMA; - register_wc (wr); - - nthread_lock (&bgpdcmf_mem_hdl->wr_queue_lock); - bgpdcmf_mem_hdl->wr_queue.push_back (wr); - nthread_unlock (&bgpdcmf_mem_hdl->wr_queue_lock); - - if (rc == NNTI_OK) - { - reg_buf->buffer_addr.transport_id = NNTI_TRANSPORT_DCMF; - memcpy (®_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.mem_hdl, - &bgpdcmf_mem_hdl->mem_hdl, sizeof (DCMF_Memregion_t)); - if (bgpdcmf_mem_hdl->type == REQUEST_BUFFER) - { - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.size = - transport_global_data.req_queue.req_size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.buf = - (uint64_t) transport_global_data.req_queue.req_buffer; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.type = - NNTI_DCMF_REQUEST_BUFFER; - } - else - { - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.size = - reg_buf->payload_size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.buf = - (uint64_t) reg_buf->payload; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.type = - NNTI_DCMF_SEND_SRC; - - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.wc_addr = - (uint64_t) & wr->wc; - memcpy (®_buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf. - wc_mem_hdl, &wr->wc_mem_hdl, sizeof (DCMF_Memregion_t)); - } - - } - - if (bgpdcmf_mem_hdl->ref_count == 1) - { - insert_buf_bufhash (reg_buf); - //log_debug (nnti_debug_level, "bgpdcmf_mem_hdl->type==%llu", - //(uint64_t) bgpdcmf_mem_hdl->type); - //log_debug (nnti_debug_level, "reg_buf.buf.hash==%llu", - //(uint64_t) hash6432shift (reg_buf->buffer_addr. - // NNTI_remote_addr_t_u.bgpdcmf.buf)); - } - - return (rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t -NNTI_bgpdcmf_unregister_memory (NNTI_buffer_t * reg_buf) -{ - NNTI_result_t - rc = NNTI_OK; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - - assert (reg_buf); - - //log_debug (nnti_debug_level, "enter unregistered buffer %p", reg_buf); - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - - assert (bgpdcmf_mem_hdl); - bgpdcmf_mem_hdl->ref_count--; - - if (bgpdcmf_mem_hdl->ref_count == 0) - { - //log_debug (nnti_debug_level, - //"bgpdcmf_mem_hdl->ref_count is 0. release all resources."); - - if (bgpdcmf_mem_hdl->type == REQUEST_BUFFER) - { - server_req_queue_destroy (&transport_global_data.req_queue); - - } - else - { - unregister_memory (bgpdcmf_mem_hdl); - } - - del_buf_bufhash (reg_buf); - nthread_lock (&bgpdcmf_mem_hdl->wr_queue_lock); - while (!bgpdcmf_mem_hdl->wr_queue.empty ()) - { - dcmf_work_request * - wr = bgpdcmf_mem_hdl->wr_queue.front (); - //log_debug (nnti_debug_level, "removing pending wr=%p", wr); - bgpdcmf_mem_hdl->wr_queue.pop_front (); - del_wr_wrhash (wr); - //log_debug (nnti_debug_level, " called in unregister memory"); - } - nthread_unlock (&bgpdcmf_mem_hdl->wr_queue_lock); - } - nthread_lock_fini (&bgpdcmf_mem_hdl->wr_queue_lock); - if (bgpdcmf_mem_hdl) - delete - bgpdcmf_mem_hdl; - - reg_buf->transport_id = NNTI_TRANSPORT_NULL; - reg_buf->payload_size = 0; - reg_buf->payload = 0; - reg_buf->transport_private = 0; - - //log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t -NNTI_bgpdcmf_put (const NNTI_buffer_t * src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t * dest_buffer_hdl, - const uint64_t dest_offset) -{ - bgpdcmf_memory_handle * - src_buf_mem_hdl = NULL; - DCMF_Protocol_t - put_prot; - NNTI_result_t - rc = NNTI_OK; - dcmf_work_request * - wr = NULL; - - wr = (dcmf_work_request *) calloc (1, sizeof (dcmf_work_request)); - assert (wr); - - wr->reg_buf = src_buffer_hdl; - - src_buf_mem_hdl = - (bgpdcmf_memory_handle *) src_buffer_hdl->transport_private; - size_t - srv_rank = - dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.owner_rank; - { - DCMF_Put_Configuration_t - put_conf = { DCMF_DEFAULT_PUT_PROTOCOL }; - DCMF_CriticalSection_enter (0); - DCMF_Put_register (&put_prot, &put_conf); - DCMF_CriticalSection_exit (0); - } - wr->peer_rank = srv_rank; - register_wc (wr); - - wr->op_state = RDMA_WRITE_INIT; - wr->wc.op = DCMF_OP_PUT_TARGET; - wr->wc.req_rank = transport_global_data.myrank; - wr->wc.byte_len = src_length; - wr->wc.src_offset = src_offset; - wr->wc.dest_offset = dest_offset; - wr->last_op = DCMF_OP_PUT_INITIATOR; - - wr->wc.local_send_complete = 1; - wr->wc.remote_put_complete = 1; - DCMF_Callback_t - cb0 = { decrement, (void *) &wr->wc.local_send_complete }; - DCMF_Callback_t - _cb_done; - DCMF_Request_t - req0; - - _cb_done.function = decrement; - _cb_done.clientdata = (void *) &wr->wc.remote_put_complete; - -/* - size_t length; - void * base; - DCMF_Memregion_query (&src_buf_mem_hdl->mem_hdl, &length, &base); - fprintf(stderr, "bgpdcmf put SRC base %p size %ld\n", base, length); - DCMF_Memregion_query ((DCMF_Memregion_t *) &dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.mem_hdl, &length, &base); - fprintf(stderr, "bgpdcmf put DEST base %p size %ld\n", base, length); -*/ - - nthread_lock (&nnti_bgpdcmf_lock); - - DCMF_CriticalSection_enter (0); - DCMF_Put (&put_prot, &req0, cb0, DCMF_SEQUENTIAL_CONSISTENCY, - srv_rank, src_length, &src_buf_mem_hdl->mem_hdl, - (DCMF_Memregion_t *) & dest_buffer_hdl->buffer_addr. - NNTI_remote_addr_t_u.bgpdcmf.mem_hdl, src_offset, dest_offset, - _cb_done); - while (wr->wc.remote_put_complete) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - //log_debug(nnti_debug_level, "messeger advance in put"); - memcpy (&wr->wc_dest_mem_hdl, - &dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf. - wc_mem_hdl, sizeof (DCMF_Memregion_t)); - nthread_unlock (&nnti_bgpdcmf_lock); - - nthread_lock (&src_buf_mem_hdl->wr_queue_lock); - src_buf_mem_hdl->wr_queue.push_back (wr); - nthread_unlock (&src_buf_mem_hdl->wr_queue_lock); - insert_wr_wrhash (wr); - //log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - */ -NNTI_result_t -NNTI_bgpdcmf_send (const NNTI_peer_t * peer_hdl, - const NNTI_buffer_t * msg_hdl, - const NNTI_buffer_t * dest_hdl) -{ - NNTI_result_t - rc = NNTI_OK; - - - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - - - assert (peer_hdl); - assert (msg_hdl); - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) msg_hdl->transport_private; - - if ((dest_hdl == NULL) || (dest_hdl->ops == NNTI_RECV_QUEUE)) - { - bgpdcmf_connection * - conn = - get_conn_rank (peer_hdl->peer.NNTI_remote_process_t_u.bgpdcmf. - pset_rank); - assert (conn); - request_send (&conn->queue_local_attrs, - &conn->queue_remote_attrs.server, msg_hdl, - conn->peer_rank); - log_debug (nnti_debug_level, "sending request to (%s)", peer_hdl->url); - - } - else - { - rc = NNTI_bgpdcmf_put (msg_hdl, 0, msg_hdl->payload_size, dest_hdl, 0); - if (rc != NNTI_OK) - log_error (nnti_debug_level, "Put() failed: %d", rc); - } - - - //log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t -NNTI_bgpdcmf_get (const NNTI_buffer_t * src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t * dest_buffer_hdl, - const uint64_t dest_offset) -{ - DCMF_Protocol_t - get_protocol; - NNTI_result_t - rc = NNTI_OK; - bgpdcmf_memory_handle * - target_buf_mem_hdl; - int - srv_rank; - dcmf_work_request * - wr = NULL; - - - wr = (dcmf_work_request *) calloc (1, sizeof (dcmf_work_request)); - wr->reg_buf = dest_buffer_hdl; - srv_rank = - src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.owner_rank; - - target_buf_mem_hdl = - (bgpdcmf_memory_handle *) dest_buffer_hdl->transport_private; - { - DCMF_Result - dcmf_result; - DCMF_Get_Configuration_t - conf; - - conf.protocol = DCMF_DEFAULT_GET_PROTOCOL; - conf.network = DCMF_TORUS_NETWORK; - - dcmf_result = DCMF_Get_register (&get_protocol, &conf); - assert (dcmf_result == DCMF_SUCCESS); - } -/* - DCMF_Memregion_t my_memregion; - DCMF_CriticalSection_enter(0); - DCMF_Result dcmf_result = DCMF_Memregion_create( &my_memregion, &bytes_out, src_length, dest_buffer_hdl->payload, 0 ); - DCMF_CriticalSection_exit(0); - assert( dcmf_result==DCMF_SUCCESS && bytes_out==src_length ); -*/ - wr->op_state = RDMA_READ_INIT; - wr->wc.op = DCMF_OP_GET_TARGET; - wr->wc.req_rank = srv_rank; - wr->wc.byte_len = src_length; - wr->wc.src_offset = src_offset; - wr->wc.dest_offset = dest_offset; - wr->last_op = DCMF_OP_GET_INITIATOR; - wr->wc.local_get_complete = 1; - wr->peer_rank = srv_rank; - register_wc (wr); - DCMF_Request_t - request; - DCMF_Callback_t - done_callback; - DCMF_Result - dcmf_result; - - size_t - regsize; - void * - base; - DCMF_Memregion_query ((DCMF_Memregion_t *)&src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf.mem_hdl, ®size, - &base); - log_debug(nnti_debug_level, "GET SRC base %p size %ld rank %d\n", base, regsize, srv_rank); - DCMF_Memregion_query (&target_buf_mem_hdl->mem_hdl, ®size, &base); - log_debug(nnti_debug_level, "GET DEST base %p size %ld length %llu\n", base, regsize, src_length); - - nthread_lock (&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - done_callback.function = decrement; - done_callback.clientdata = (void *) &wr->wc.local_get_complete; - dcmf_result = DCMF_Get (&get_protocol, - &request, - done_callback, - DCMF_SEQUENTIAL_CONSISTENCY, - srv_rank, - src_length, - (DCMF_Memregion_t *) & src_buffer_hdl->buffer_addr. - NNTI_remote_addr_t_u.bgpdcmf.mem_hdl, - &target_buf_mem_hdl->mem_hdl, src_offset, - dest_offset); - while (wr->wc.local_get_complete > 0) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - log_debug(nnti_debug_level, "messager advance in get"); - assert (dcmf_result == DCMF_SUCCESS); - memcpy (&wr->wc_dest_mem_hdl, - &src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf. - wc_mem_hdl, sizeof (DCMF_Memregion_t)); - nthread_unlock (&nnti_bgpdcmf_lock); - nthread_lock (&target_buf_mem_hdl->wr_queue_lock); - target_buf_mem_hdl->wr_queue.push_back (wr); - nthread_unlock (&target_buf_mem_hdl->wr_queue_lock); - insert_wr_wrhash (wr); - //log_debug (nnti_debug_level, "exit"); - - return (rc); - -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t -NNTI_bgpdcmf_wait (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - const int timeout, NNTI_status_t * status) -{ - NNTI_result_t - nnti_rc = NNTI_OK; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - dcmf_work_request * - wr = NULL; - bgpdcmf_connection * - conn = NULL; - int - timeout_per_call; - uint8_t - retry_count = 0; - - - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - - if (timeout < 0) - timeout_per_call = MIN_TIMEOUT; - else - timeout_per_call = (timeout < MIN_TIMEOUT) ? MIN_TIMEOUT : timeout; - - retry_count = 0; - wr = bgpdcmf_mem_hdl->wr_queue.front (); - bgpdcmf_mem_hdl->wr_queue.pop_front (); - if ((wr->last_op == DCMF_OP_REGISTER_RDMA)) - { - if ((remote_op == NNTI_SEND_SRC) || (remote_op == NNTI_GET_DST) - || (remote_op == NNTI_PUT_SRC)) - { - wr = bgpdcmf_mem_hdl->wr_queue.front (); - bgpdcmf_mem_hdl->wr_queue.pop_front (); - } - } - log_debug (nnti_debug_level, "bgpdcmf_wait buffer = %p work request =%p", reg_buf, wr); - nnti_rc = process_event (reg_buf, remote_op, wr, timeout_per_call); - conn = get_conn_rank (wr->peer_rank); - status->op = remote_op; - status->result = nnti_rc; - if (nnti_rc == NNTI_OK) - { - status->start = (uint64_t) reg_buf->payload; - status->offset = wr->wc.byte_offset; - status->length = wr->wc.byte_len; - switch (remote_op) - { - case NNTI_SEND_SRC: - case NNTI_GET_SRC: - case NNTI_PUT_SRC: /* I am client here */ - create_peer (&status->src, transport_global_data.myrank); - create_peer (&status->dest, conn->peer_rank); - break; - case NNTI_RECV_QUEUE: - case NNTI_RECV_DST: - case NNTI_GET_DST: - case NNTI_PUT_DST: - create_peer (&status->src, conn->peer_rank); - create_peer (&status->dest, transport_global_data.myrank); - break; - - } - } - if (nnti_rc == NNTI_OK) - { - - if ((bgpdcmf_mem_hdl->type == RDMA_TARGET_BUFFER) - || (bgpdcmf_mem_hdl->type == RECEIVE_BUFFER) - || (bgpdcmf_mem_hdl->type == RESULT_BUFFER) - || (bgpdcmf_mem_hdl->type == GET_SRC_BUFFER) - || (bgpdcmf_mem_hdl->type == REQUEST_BUFFER) - || (bgpdcmf_mem_hdl->type == PUT_DST_BUFFER)) - { - repost_recv_work_request((NNTI_buffer_t *)reg_buf, wr); - } - else - { - del_wr_wrhash (wr); - log_debug(nnti_debug_level, " delete wr called in bgpdcmf_wait wr = %p buffer = %p", wr, reg_buf); - free (wr); - } - } - - - return (nnti_rc); -} - -/** - * @brief Wait for remote_op on any buffer in buf_list to complete. - * - * Wait for remote_op on any buffer in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the request queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t -NNTI_bgpdcmf_waitany (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, - uint32_t * which, NNTI_status_t * status) -{ - NNTI_result_t - nnti_rc = NNTI_OK; - - log_level - debug_level = nnti_debug_level; - - log_debug (debug_level, "enter"); - - assert (buf_list); - assert (buf_count > 0); - if (buf_count > 1) - { - /* if there is more than 1 buffer in the list, none of them can be a REQUEST_BUFFER */ - for (uint32_t i = 0; i < buf_count; i++) - { - if (buf_list[i] != NULL) - { - assert (((bgpdcmf_memory_handle *) buf_list[i]-> - transport_private)->type != REQUEST_BUFFER); - } - } - } - assert (status); - - if (buf_count == 1) - { - nnti_rc = NNTI_bgpdcmf_wait (buf_list[0], remote_op, timeout, status); - *which = 0; - goto cleanup; - } - - while (1) - { - if (is_any_buf_op_complete (buf_list, buf_count, which) == TRUE) - { - log_debug (debug_level, - "buffer op already complete (which=%u, buf_list[%d]=%p)", - *which, *which, buf_list[*which]); - nnti_rc = NNTI_OK; - } - else - { - log_debug (debug_level, "buffer op NOT complete (buf_list=%p)", - buf_list); - - for (uint32_t i = 0; i < buf_count; i++) - { - if (buf_list[i] != NULL) - { - nnti_rc = - NNTI_bgpdcmf_wait (buf_list[i], remote_op, - timeout / buf_count, status); - if (nnti_rc == NNTI_OK) - { - *which = i; - break; - } - } - } - - } - } - -cleanup: - - //log_debug (debug_level, "exit"); - - trios_stop_timer ("NNTI_bgpdcmf_waitany", total_time); - - return (nnti_rc); -} - -NNTI_result_t -NNTI_bgpdcmf_waitall (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, NNTI_status_t ** status) -{ - return NNTI_OK; -} - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t -NNTI_bgpdcmf_fini (const NNTI_transport_t * trans_hdl) -{ - /* Free up injection counter and global inj fifo and reception FIFO */ - DCMF_Messager_finalize (); - return (NNTI_OK); -} - -static NNTI_result_t -register_wc (dcmf_work_request * wr) -{ - DCMF_Result - dcmf_result; - size_t - bytes_out; - - -/* This is for Work completion memory handle where ACK is received after PUT/GET */ - dcmf_result = - DCMF_Memregion_create ((DCMF_Memregion_t *) & wr->wc_mem_hdl, &bytes_out, - sizeof (nnti_bgpdcmf_work_completion), &wr->wc, 0); - - if (dcmf_result != DCMF_SUCCESS) - { - fprintf (stderr, - "DCMF memregion create failed in register memory Work completion handle\n"); - return (NNTI_EIO); - } - - //log_debug (nnti_debug_level, "exit wr(%p)", wr); - - return ((NNTI_result_t) DCMF_SUCCESS); - -} - - -static - NNTI_result_t -register_memory (bgpdcmf_memory_handle * hdl, void *buf, uint64_t len, - const NNTI_buf_ops_t remote_op) -{ - NNTI_result_t - rc = NNTI_OK; /* return code */ - DCMF_Result - dcmf_result; - size_t - bytes_out; - - assert (hdl); - dcmf_result = - DCMF_Memregion_create (&hdl->mem_hdl, &bytes_out, len, buf, 0); - if (dcmf_result != DCMF_SUCCESS) - fprintf (stderr, "DCMF memregion create failed in register memory \n"); - - return (rc); -} - -static int -unregister_memory (bgpdcmf_memory_handle * hdl) -{ - int - rc = NNTI_OK; /* return code */ - - log_debug (nnti_debug_level, "exit hdl(%p)", hdl); - - return (rc); -} - -static - NNTI_result_t -send_ack (const NNTI_buffer_t * reg_buf, dcmf_work_request * wr) -{ - NNTI_result_t - rc = NNTI_OK; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - DCMF_Protocol_t - put_prot; - - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; -#ifndef HAS_MPI - DCMF_Messager_initialize (); -#endif - { /* init put */ - DCMF_Put_Configuration_t - put_conf = { DCMF_DEFAULT_PUT_PROTOCOL }; - DCMF_CriticalSection_enter (0); - DCMF_Put_register (&put_prot, &put_conf); - DCMF_CriticalSection_exit (0); - } - nthread_lock(&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - volatile unsigned - active0 = 1; - DCMF_Callback_t - cb0 = { decrement, (void *) &active0 }; - DCMF_Callback_t - _cb_null; - DCMF_Request_t - req0; - - _cb_null.function = NULL; - _cb_null.clientdata = (void *) NULL; - - /* SHYAMALI - size_t length; - void * base; - DCMF_Memregion_query (&wr->wc_mem_hdl, &length, &base); - DCMF_Memregion_query ((DCMF_Memregion_t *) &wr->wc_dest_mem_hdl, &length, &base); - */ - - DCMF_Put (&put_prot, &req0, cb0, DCMF_SEQUENTIAL_CONSISTENCY, - wr->peer_rank, sizeof (nnti_bgpdcmf_work_completion), - &wr->wc_mem_hdl, &wr->wc_dest_mem_hdl, 0, 0, _cb_null); - while (active0) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - nthread_unlock(&nnti_bgpdcmf_lock); - //log_debug(nnti_debug_level, "messager advance in send ack"); - return (rc); -} - - -static - NNTI_result_t -process_event (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - dcmf_work_request * wr, const int timeout) -{ - NNTI_result_t - nnti_rc = NNTI_OK; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - - log_level - debug_level = nnti_debug_level; - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - - //log_debug (nnti_debug_level, "enter"); - long - entry_time = trios_get_time_ms (); - long - elapsed_time = 0; - - - debug_level = nnti_debug_level; - switch (bgpdcmf_mem_hdl->type) - { - case SEND_BUFFER: - while (wr->wc.remote_put_complete > 0) - { - usleep(100); - log_debug(nnti_debug_level, "messager advance in process event"); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out SEND BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - } - wr->op_state = SEND_COMPLETE; - wr->wc.op_complete = 1; - if (wr->wc.op != DCMF_OP_SEND_REQ) - { - fprintf (stderr, "process_event send buffer sending ack\n"); - send_ack (reg_buf, wr); - } - log_debug (nnti_debug_level, "process event send buffer"); - break; - case PUT_SRC_BUFFER: - wr->last_op = DCMF_OP_PUT_INITIATOR; - if (wr->op_state == RDMA_WRITE_INIT) - { - while (wr->wc.remote_put_complete > 0) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out PUT SRC BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->wc.op_complete = 1; - wr->op_state = RDMA_COMPLETE; - log_debug (nnti_debug_level, "process event put src buffer"); - send_ack (reg_buf, wr); - } - break; - case GET_DST_BUFFER: - wr->last_op = DCMF_OP_GET_INITIATOR; - if (wr->op_state == RDMA_READ_INIT) - { - while (wr->wc.local_get_complete > 0) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out GET DST BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - wr->wc.op_complete = 1; - log_debug (nnti_debug_level, "process event get dest buffer"); - send_ack (reg_buf, wr); - } - break; - case REQUEST_BUFFER: - { - uint64_t - index = 0; - bgpdcmf_request_queue_handle * - q = &transport_global_data.req_queue; - - wr->last_op = DCMF_OP_NEW_REQUEST; - - log_debug(nnti_debug_level, - "recv completion - reg_buf=%p current_req_index =%llu processing=%llu\n", - reg_buf, q->req_index, q->req_processed); - while (q->req_index == q->req_processed) - { - /* elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out REQUEST BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - */ - usleep(100); - - } - index = q->req_processed; - /* Shyamali wait for work completion buffer to showup */ - while (q->wc_buffer[q->req_processed].req_received != 1) - { - usleep (100); - } - - q->wc_buffer[q->req_processed].byte_offset = index * q->req_size; - q->wc_buffer[q->req_processed].byte_len = q->req_size; - wr->wc = q->wc_buffer[q->req_processed]; - wr->peer_rank = q->wc_buffer[q->req_processed].req_rank; - fprintf (stderr, "This request came from %d process index %llu\n", - wr->peer_rank, q->req_processed); - wr->op_state = RECV_COMPLETE; - q->req_processed++; - q->total_req_processed++; - if (q->req_processed > q->req_count) - { - log_error (nnti_debug_level, - "req_processed(%llu) > req_count(%llu) fail", - q->req_processed, q->req_count); - } - if (q->req_processed == (q->req_count / 2)) - { - if (q->req_index >= q->req_count) - { - reset_req_index (q); - log_debug (nnti_debug_level, - "resetting req_processed(%llu) total_req_processed(%llu)", - q->req_processed, q->total_req_processed); - } - else - { - log_debug (nnti_debug_level, - "skipping reset req_processed(%llu) total_req_processed(%llu)", - q->req_processed, q->total_req_processed); - } - } - if (q->req_processed == q->req_processed_reset_limit) - { - q->req_processed = 0; - } - log_debug (nnti_debug_level, - "current req_processed(%llu) req_count(%llu)", - q->req_processed, q->req_count); - } - break; - case RECEIVE_BUFFER: - wr->last_op = DCMF_OP_RECEIVE; - log_debug (debug_level, - "receive buffer - recv completion - reg_buf==%p", reg_buf); - - while (wr->wc.op_complete != 1) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out TARGET RECEIVE BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->peer_rank = wr->wc.req_rank; - wr->op_state = RECV_COMPLETE; - break; - case RESULT_BUFFER: - wr->last_op = DCMF_OP_PUT_TARGET; - while (wr->wc.op_complete != 1) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out RESULT BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->peer_rank = wr->wc.req_rank; - wr->op_state = RDMA_COMPLETE; - log_debug(nnti_debug_level, "process event result buffer"); - break; - case PUT_DST_BUFFER: - wr->last_op = DCMF_OP_PUT_TARGET; - //fprintf (stderr, "PUT DST Buffer\n"); - while (wr->wc.op_complete != 1) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out PUT DST BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - log_debug(nnti_debug_level, "process event PUT DST buffer"); - break; - case GET_SRC_BUFFER: - wr->last_op = DCMF_OP_GET_TARGET; - //fprintf (stderr, "GET SRC BUFFER\n"); - if (wr->op_state == RDMA_READ_INIT) - { - while (wr->wc.op_complete != 1) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out GET SRC BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - } - log_debug(nnti_debug_level, "process event GET SRC buffer"); - break; - case RDMA_TARGET_BUFFER: - if ((wr->last_op == DCMF_OP_GET_INITIATOR) || - (wr->last_op == DCMF_OP_PUT_INITIATOR)) - { - - if (wr->op_state == RDMA_TARGET_INIT) - { - while (wr->wc.op_complete != 1) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out RDMA TARGET BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - - } - - wr->op_state = RDMA_COMPLETE; - - } - } - else - { - if (wr->op_state == RDMA_TARGET_INIT) - { - - log_debug (debug_level, "RDMA target completion - reg_buf==%p", - reg_buf); - while (wr->wc.op_complete != 1) - { - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out RDMA BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - - } - } - - break; - case UNKNOWN_BUFFER: - fprintf (stderr, "UNKNOWN_BUFFER\n"); - break; - } - //log_debug (nnti_debug_level, "exit"); - return (nnti_rc); -} - - -static void -create_peer (NNTI_peer_t * peer, int rank) -{ - sprintf (peer->url, "dcmf://%d", rank); - - peer->peer.transport_id = NNTI_TRANSPORT_DCMF; - peer->peer.NNTI_remote_process_t_u.bgpdcmf.pset_rank = rank; -} - - -/* - * Keep looping until all bytes have been accepted by the kernel. - */ -static - NNTI_result_t -inject_full (int rank, const void *buf, size_t num) -{ - NNTI_result_t - rc = NNTI_OK; - DCQuad - msginfo; - DCMF_Request_t - request; - - send_active = 1; - nthread_lock(&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - DCMF_Callback_t - cb_info = { decrement, (void *) &send_active }; - DCMF_Send (&send_prot, - &request, - cb_info, - DCMF_SEQUENTIAL_CONSISTENCY, - rank, num, (char *) buf, &msginfo, 1); - - TRACE_ERR ((stderr, "inject to (%zd) Before advance size = %d\n", rank, - num)); - while (send_active) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - nthread_unlock(&nnti_bgpdcmf_lock); - send_active = 1; - return rc; -} - -static - NNTI_result_t -new_client_connection (bgpdcmf_connection * c, int peer_rank) -{ - NNTI_result_t - rc; - - /* - * Values passed DMA MEMFIO for initial connection. - */ - struct - { - nnti_bgpdcmf_server_queue_attrs - server_attrs; - } sa_in; - struct - { - nnti_bgpdcmf_client_queue_attrs - client_attrs; - } ca_out; - - - c->connection_type = CLIENT_CONNECTION; - c->peer_rank = peer_rank; - memset (&sa_in, 0, sizeof (sa_in)); - - _recv_active = 1; - confirm_conn = 1; - nthread_lock(&nnti_bgpdcmf_lock); - while (_recv_active) - DCMF_Messager_advance (); - nthread_unlock(&nnti_bgpdcmf_lock); - memcpy (&sa_in, &_recv_buffer, sizeof (sa_in)); - _recv_active = 1; - - c->queue_remote_attrs.server = sa_in.server_attrs; - client_req_queue_init (c); - memset (&ca_out, 0, sizeof (ca_out)); - ca_out.client_attrs.req_index = c->queue_local_attrs.req_index; - ca_out.client_attrs.req_index_addr = c->queue_local_attrs.req_index_addr; - memcpy (&ca_out.client_attrs.req_index_mem_hdl, - &c->queue_local_attrs.req_index_mem_hdl, sizeof (DCMF_Memregion_t)); - - rc = inject_full (peer_rank, &ca_out, sizeof (ca_out)); - nthread_lock(&nnti_bgpdcmf_lock); - while (confirm_conn) - DCMF_Messager_advance (); - nthread_unlock(&nnti_bgpdcmf_lock); - return rc; -} - -static - NNTI_result_t -new_server_connection (bgpdcmf_connection * c, int rank) -{ - NNTI_result_t - rc; - bgpdcmf_request_queue_handle * - q_hdl = &transport_global_data.req_queue; - - /* - * Values passed through DMA MEMFIFO interface to permit initial connection. - */ - struct - { - nnti_bgpdcmf_server_queue_attrs - server_attrs; - } sa_out; - - assert (transport_global_data.req_queue.reg_buf); - - c->connection_type = SERVER_CONNECTION; - c->peer_rank = rank; - - memset (&sa_out, 0, sizeof (sa_out)); - - sa_out.server_attrs.req_index_addr = - transport_global_data.req_queue.req_index_addr; - - memcpy (&sa_out.server_attrs.req_index_mem_hdl, - &transport_global_data.req_queue.req_index_mem_hdl, - sizeof (DCMF_Memregion_t)); - - sa_out.server_attrs.req_buffer_addr = - (uint64_t) transport_global_data.req_queue.req_buffer; - sa_out.server_attrs.req_size = transport_global_data.req_queue.req_size; - sa_out.server_attrs.req_count = transport_global_data.req_queue.req_count; - memcpy (&sa_out.server_attrs.req_mem_hdl, q_hdl->req_mem_hdl, - sizeof (DCMF_Memregion_t)); - sa_out.server_attrs.wc_buffer_addr = - (uint64_t) transport_global_data.req_queue.wc_buffer; - memcpy (&sa_out.server_attrs.wc_mem_hdl, - &transport_global_data.req_queue.wc_mem_hdl, - sizeof (DCMF_Memregion_t)); - sa_out.server_attrs.unblock_buffer_addr = - (uint64_t) transport_global_data.req_queue.unblock_buffer; - memcpy (&sa_out.server_attrs.unblock_mem_hdl, - &transport_global_data.req_queue.unblock_mem_hdl, - sizeof (DCMF_Memregion_t)); - rc = inject_full (rank, &sa_out, sizeof (sa_out)); - if (rc) - { - fprintf (stderr, "new server connection: failed inject full\n"); - goto out; - } - -out: - return rc; -} - -static - NNTI_result_t -insert_conn_rank (const uint32_t pset_rank, bgpdcmf_connection * conn) -{ - NNTI_result_t - rc = NNTI_OK; - - nthread_lock (&nnti_conn_peer_lock); - if (connections_by_rank.find (pset_rank) != connections_by_rank.end ()) - { - if (connections_by_rank[pset_rank] == conn){ - log_debug (nnti_debug_level, "connection already exists"); - return(rc); - } - } - connections_by_rank[pset_rank] = conn; // add to connection map - nthread_unlock (&nnti_conn_peer_lock); - - log_debug (nnti_debug_level, "peer connection added (conn=%p)", conn); - - return (rc); -} - -static bgpdcmf_connection * -get_conn_rank (const uint32_t pset_rank) -{ - - int - key = pset_rank; - bgpdcmf_connection * - conn = NULL; - - nthread_lock (&nnti_conn_peer_lock); - if (connections_by_rank.find (key) != connections_by_rank.end ()) - { - conn = connections_by_rank[key]; - } - - nthread_unlock (&nnti_conn_peer_lock); - - if (conn != NULL) - { - return conn; - } - log_debug (nnti_debug_level, "connection NOT found"); - return (NULL); -} - -static bgpdcmf_connection * -del_conn_rank (const uint32_t pset_rank) -{ - bgpdcmf_connection * - conn = NULL; - int - key; - - - key = pset_rank; - - nthread_lock (&nnti_conn_peer_lock); - if (connections_by_rank.find (key) != connections_by_rank.end ()) - { - conn = connections_by_rank[key]; - } - - nthread_unlock (&nnti_conn_peer_lock); - - if (conn != NULL) - { - connections_by_rank.erase (key); - } - else - { - log_debug (nnti_debug_level, "connection NOT found"); - } - - return (conn); -} - -/** - * @brief initialize - */ -static - NNTI_result_t -init_connection (bgpdcmf_connection ** conn, - int peer_rank, const int is_server) -{ - NNTI_result_t - rc = NNTI_OK; /* return code */ - - bgpdcmf_connection * - c = NULL; - - - c = (bgpdcmf_connection *) calloc (1, sizeof (bgpdcmf_connection)); - if (c == NULL) - { - log_error (nnti_debug_level, - "calloc returned NULL. out of memory?: %s", - strerror (errno)); - rc = NNTI_ENOMEM; - goto out; - } - - if (is_server) - { - rc = new_server_connection (c, peer_rank); - } - else - { - rc = new_client_connection (c, peer_rank); - } - if (rc) - { - close_connection (c); - c = NULL; - goto out; - } - - *conn = c; - -out: - return (rc); -} - -/* - * At an explicit BYE message, or at finalize time, shut down a connection. - * If descriptors are posted, defer and clean up the connection structures - * later. - */ -static void -close_connection (bgpdcmf_connection * c) -{ - - if (c == NULL) - return; - - log_debug (nnti_debug_level, "enter"); - - if (c->connection_type == CLIENT_CONNECTION) - { - client_req_queue_destroy (c); - } - - c->state = DISCONNECTED; - - log_debug (nnti_debug_level, "exit"); -} - -/** - * Check for new connections. The listening socket is left nonblocking - * so this test can be quick; but accept is not really that quick compared - * to polling an Gemini interface, for instance. Returns >0 if an accept worked. - */ -static - NNTI_result_t -check_poll_for_new_connections () -{ - NNTI_result_t - rc = NNTI_OK; - - bgpdcmf_connection * - conn = NULL; - uint32_t - peer_rank; - int - i; - - _recv_active = 1; - _is_server = 1; - log_debug(nnti_debug_level, "server poll for new connection"); - while (_recv_active) - { - - nthread_lock (&nnti_bgpdcmf_lock); - for (i = 0; i< 1024; ++i) - { - peer_rank = transport_global_data.conn_req[i]; - if(peer_rank != -1) { - conn = get_conn_rank(peer_rank); - if (conn == NULL) { - rc = init_connection (&conn, peer_rank, 1); - if (rc != NNTI_OK) - { - fprintf (stderr, "Server make new connection failed\n"); - } - insert_conn_rank (peer_rank, conn); - } - } - } - DCMF_Messager_advance (); - nthread_unlock(&nnti_bgpdcmf_lock); - usleep(10000); - } -cleanup: - return rc; -} - -/** - * @brief Continually check for new connection attempts. - * - */ -static void * -connection_listener_thread (void *args) -{ - NNTI_result_t - rc = NNTI_OK; - - log_debug (nnti_debug_level, - "started thread to listen for client connection attempts"); - - /* SIGINT (Ctrl-C) will get us out of this loop */ - while (1) - { - log_debug (nnti_debug_level, "listening for new connection"); - rc = check_poll_for_new_connections (); - if (rc != NNTI_OK) - { - log_fatal (nnti_debug_level, - "error returned from nssi_bgpdcmf_server_listen_for_client: %d", - rc); - continue; - } - } - - return (NULL); -} - -static void * -fetch_req_index_thread (void *args) -{ - int - i; - size_t - peer; - DCMF_Protocol_t - put_prot; - bgpdcmf_request_queue_handle * - q = &transport_global_data.req_queue; - - log_debug (nnti_debug_level, "started thread to service req_index fetch"); - { - DCMF_Put_Configuration_t - put_conf = { DCMF_DEFAULT_PUT_PROTOCOL }; - DCMF_CriticalSection_enter (0); - DCMF_Put_register (&put_prot, &put_conf); - DCMF_CriticalSection_exit (0); - } - volatile unsigned - active0 = 1; - DCMF_Callback_t - cb0 = { decrement, (void *) &active0 }; - DCMF_Callback_t - _cb_null; - DCMF_Request_t - req0; - _cb_null.function = NULL; - _cb_null.clientdata = (void *) NULL; - - - /* SIGINT (Ctrl-C) will get us out of this loop */ - log_debug (nnti_debug_level, "listening for buffer index to send request"); - while (1) - { - for (i = 0; i < MAX_CONNECTION; ++i) - { - if (q->unblock_buffer[i].req_rank == i) - { - nthread_lock (&nnti_index_lock); - peer = q->unblock_buffer[i].req_rank; - //fprintf (stderr, " fetch index rank %d index %d\n", peer, i); - DCMF_CriticalSection_enter (0); - if (peer != -1) - { - active0 = 1; - - DCMF_Put (&put_prot, &req0, cb0, - DCMF_SEQUENTIAL_CONSISTENCY, peer, - sizeof (uint64_t), &q->req_index_mem_hdl, - (DCMF_Memregion_t *) & q->unblock_buffer[i]. - response_hdl, 0, 0, _cb_null); - while (active0) - DCMF_Messager_advance (); - q->req_index++; - q->unblock_buffer[i].req_rank = -1; - } - DCMF_CriticalSection_exit (0); - nthread_unlock (&nnti_index_lock); - } - } - } - - return (NULL); -} - - -/** - * @brief Start a thread to check for new connection attempts. - * - */ -static int -start_connection_listener_thread () -{ - int - rc = 0; - pthread_t - thread; - - /* Create the thread. Do we want special attributes for this? */ - rc = pthread_create (&thread, NULL, connection_listener_thread, NULL); - if (rc) - { - log_error (nnti_debug_level, - "could not spawn thread for new connection"); - rc = NNTI_EBADRPC; - } - return rc; - -} - -static int -start_index_thread () -{ - int - rc = 0; - pthread_t - thread1; - rc = pthread_create (&thread1, NULL, fetch_req_index_thread, NULL); - if (rc) - { - log_error (nnti_debug_level, - "could not spawn thread for fetch_req buffer index"); - rc = NNTI_EBADRPC; - } - - return rc; -} - - - -static int -reset_req_index (bgpdcmf_request_queue_handle * req_queue_attrs) -{ - uint64_t - value_before_reset = 0; - -/* Do a remote get or Direct put to update req_index */ - - req_queue_attrs->last_index_before_reset = value_before_reset; - req_queue_attrs->req_processed_reset_limit = value_before_reset; - req_queue_attrs->req_index = 0; - log_debug (nnti_debug_level, "index before reset(%llu).", - req_queue_attrs->last_index_before_reset); - log_debug (nnti_debug_level, "index after reset(%llu).", - req_queue_attrs->req_index); - - - return (0); -} - -static int -fetch_server_req_buffer_offset (nnti_bgpdcmf_client_queue_attrs * - local_req_queue_attrs, - nnti_bgpdcmf_server_queue_attrs * - remote_req_queue_attrs, uint64_t addend, - uint64_t * prev_offset, int srv_rank) -{ - - - uint32_t - req_size = sizeof (nnti_bgpdcmf_fetch_index_req); - DCMF_Protocol_t - put_prot; - { - DCMF_Put_Configuration_t - put_conf = { DCMF_DEFAULT_PUT_PROTOCOL }; - DCMF_CriticalSection_enter (0); - DCMF_Put_register (&put_prot, &put_conf); - DCMF_CriticalSection_exit (0); - } - - local_req_queue_attrs->req_index = -1; - volatile unsigned - active0 = 1; - DCMF_Callback_t - cb0 = { decrement, (void *) &active0 }; - DCMF_Callback_t - _cb_null; - DCMF_Request_t - req0; - - _cb_null.function = NULL; - _cb_null.clientdata = (void *) NULL; - - size_t - length; - void * - base; - DCMF_Memregion_query (&local_req_queue_attrs->fetch_index_hdl, &length, - &base); - nthread_lock(&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - - DCMF_Put (&put_prot, &req0, cb0, DCMF_SEQUENTIAL_CONSISTENCY, - srv_rank, sizeof (nnti_bgpdcmf_fetch_index_req), - &local_req_queue_attrs->fetch_index_hdl, - (DCMF_Memregion_t *) & remote_req_queue_attrs->unblock_mem_hdl, 0, - req_size * transport_global_data.myrank, _cb_null); - while (active0) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - nthread_unlock(&nnti_bgpdcmf_lock); - - while (local_req_queue_attrs->req_index == -1) - usleep (10); - *prev_offset = local_req_queue_attrs->req_index; - return (0); -} - -static int -send_req (nnti_bgpdcmf_client_queue_attrs * local_req_queue_attrs, - nnti_bgpdcmf_server_queue_attrs * remote_req_queue_attrs, - uint64_t offset, const NNTI_buffer_t * reg_buf, int rank, - dcmf_work_request * wr) -{ - bgpdcmf_memory_handle * - src_buf_mem_hdl = NULL; - src_buf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - uint64_t - length; - DCMF_Protocol_t - put_prot; - { - DCMF_Put_Configuration_t - put_conf = { DCMF_DEFAULT_PUT_PROTOCOL }; - DCMF_CriticalSection_enter (0); - DCMF_Put_register (&put_prot, &put_conf); - DCMF_CriticalSection_exit (0); - } - length = reg_buf->payload_size; - { - wr->wc.remote_put_complete = 1; /* callback will decr when recv done on remote side */ - wr->wc.local_send_complete = 1; /* First callback returns after injected into local TORUS */ - DCMF_Callback_t - cb0 = { decrement, (void *) &wr->wc.local_send_complete }; - DCMF_Callback_t - _cb_done; - DCMF_Request_t - req0; - - _cb_done.function = decrement; - _cb_done.clientdata = (void *) &wr->wc.remote_put_complete; -/* - size_t - regsize; - void * - base; - DCMF_Memregion_query (&remote_req_queue_attrs->req_mem_hdl, ®size, - &base); - log_debug(nnti_debug_level, "remote req send base %p size %ld\n", base, regsize); - DCMF_Memregion_query (&src_buf_mem_hdl->mem_hdl, ®size, &base); - log_debug(nnti_debug_level, "local buffer req send base %p size %ld length %llu\n", base, regsize, length); - -*/ - nthread_lock(&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - DCMF_Put (&put_prot, &req0, cb0, DCMF_SEQUENTIAL_CONSISTENCY, - rank, length, &src_buf_mem_hdl->mem_hdl, - &remote_req_queue_attrs->req_mem_hdl, 0, offset, _cb_done); - while (wr->wc.remote_put_complete) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - nthread_unlock(&nnti_bgpdcmf_lock); - } - return (0); -} - -static int -send_req_wc (nnti_bgpdcmf_client_queue_attrs * client_q, - nnti_bgpdcmf_server_queue_attrs * server_q, - const NNTI_buffer_t * reg_buf, uint64_t offset, - dcmf_work_request * wr, int rank) -{ - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - DCMF_Protocol_t - put_prot; - { - DCMF_Put_Configuration_t - put_conf = { DCMF_DEFAULT_PUT_PROTOCOL }; - DCMF_CriticalSection_enter (0); - DCMF_Put_register (&put_prot, &put_conf); - DCMF_CriticalSection_exit (0); - } - - { - volatile unsigned - active0 = 1; - DCMF_Callback_t - cb0 = { decrement, (void *) &active0 }; - DCMF_Callback_t - _cb_null; - DCMF_Request_t - req0; - - _cb_null.function = NULL; - _cb_null.clientdata = (void *) NULL; - register_wc (wr); -/* - SHYAMALI DEBUG - if(transport_global_data.myrank == 0) { - size_t regsize; - void * base; - DCMF_Memregion_query (&server_q->wc_mem_hdl, ®size, &base); - fprintf(stderr, "server work completion base %p size %ld\n", base, regsize); - DCMF_Memregion_query (&wr->wc_mem_hdl, ®size, &base); - fprintf(stderr, "client work completion base %p size %ld\n", base, regsize); - } - */ - nthread_lock(&nnti_bgpdcmf_lock); - DCMF_CriticalSection_enter (0); - DCMF_Put (&put_prot, &req0, cb0, DCMF_SEQUENTIAL_CONSISTENCY, - rank, sizeof (nnti_bgpdcmf_work_completion), &wr->wc_mem_hdl, - &server_q->wc_mem_hdl, 0, offset, _cb_null); - while (active0) - DCMF_Messager_advance (); - DCMF_CriticalSection_exit (0); - nthread_unlock(&nnti_bgpdcmf_lock); - } - return (0); - -} - - - -static int -request_send (nnti_bgpdcmf_client_queue_attrs * client_q, - nnti_bgpdcmf_server_queue_attrs * server_q, - const NNTI_buffer_t * reg_buf, int rank) -{ - int - rc = 0; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - uint64_t - offset; - dcmf_work_request * - wr = NULL; - - uint32_t - wc_size = sizeof (nnti_bgpdcmf_work_completion); - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - - assert (bgpdcmf_mem_hdl); - wr = (dcmf_work_request *) malloc (sizeof (dcmf_work_request)); - memset (wr, 0, sizeof (dcmf_work_request)); - assert (wr); - - wr->reg_buf = reg_buf; - - bgpdcmf_mem_hdl->wr_queue.push_back (wr); - - insert_wr_wrhash (wr); - rc = fetch_server_req_buffer_offset (client_q, server_q, 1, &offset, rank); - - rc = - send_req (client_q, server_q, offset * server_q->req_size, reg_buf, rank, - wr); - if (rc != NNTI_OK) - log_error (nnti_debug_level, "send_req() failed: %d", rc); - wr->wc.op = DCMF_OP_SEND_REQ; - wr->wc.req_received = 1; - wr->wc.req_rank = transport_global_data.myrank; - wr->wc.byte_len = reg_buf->payload_size; - wr->wc.byte_offset = client_q->req_index * server_q->req_size; - wr->wc.src_offset = 0; - wr->wc.dest_offset = client_q->req_index * server_q->req_size; - wr->peer_rank = rank; - - log_debug (nnti_debug_level, "calling send_req_wc()"); - rc = send_req_wc (client_q, server_q, reg_buf, offset * wc_size, wr, rank); - return (0); -} - - - -static int -client_req_queue_init (bgpdcmf_connection * c) -{ - int - rc = 0; - size_t - bytes; - - nnti_bgpdcmf_client_queue_attrs * - q = &c->queue_local_attrs; - q->req_index = 0; - q->req_index_addr = (uint64_t) & q->req_index; - DCMF_Memregion_create (&q->req_index_mem_hdl, &bytes, - sizeof (uint64_t), (void *) q->req_index_addr, 0); - client_fetch_req.req_rank = transport_global_data.myrank; - memcpy (&client_fetch_req.response_hdl, &q->req_index_mem_hdl, - sizeof (DCMF_Memregion_t)); - DCMF_Memregion_create (&q->fetch_index_hdl, &bytes, - sizeof (nnti_bgpdcmf_fetch_index_req), - (void *) &client_fetch_req, 0); - return (rc); -} - -static int -client_req_queue_destroy (bgpdcmf_connection * c) -{ - return (0); -} - -static int -server_req_queue_init (bgpdcmf_request_queue_handle * q, - char *buffer, uint64_t req_size, uint64_t req_count) -{ - int - i; - size_t - bytes; - uint64_t - unblock_size; - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) q->reg_buf->transport_private; - - - q->req_buffer = buffer; - q->req_size = req_size; - q->req_count = req_count; - q->req_buffer_size = req_count * req_size; - - q->wc_buffer_size = q->req_count * sizeof (nnti_bgpdcmf_work_completion); - q->wc_buffer = - (nnti_bgpdcmf_work_completion *) calloc (q->req_count, - sizeof - (nnti_bgpdcmf_work_completion)); - - unblock_size = MAX_CONNECTION * sizeof (nnti_bgpdcmf_work_completion); - q->unblock_buffer = - (nnti_bgpdcmf_fetch_index_req *) calloc (MAX_CONNECTION, - sizeof - (nnti_bgpdcmf_fetch_index_req)); - DCMF_Memregion_create (&q->req_mem_hdl, &bytes, q->req_buffer_size, buffer, - 0); - DCMF_Memregion_create (&q->wc_mem_hdl, &bytes, q->wc_buffer_size, - q->wc_buffer, 0); - DCMF_Memregion_create (&q->unblock_mem_hdl, &bytes, unblock_size, - q->unblock_buffer, 0); - q->req_index = 0; - q->req_index_addr = (uint64_t) & q->req_index; - - q->req_processed = 0; - q->req_processed_reset_limit = q->req_count; - bgpdcmf_mem_hdl->type = REQUEST_BUFFER; - DCMF_Memregion_create (&q->req_index_mem_hdl, &bytes, - sizeof (uint64_t), (void *) q->req_index_addr, 0); - for (i = 0; i < MAX_CONNECTION; ++i) - q->unblock_buffer[i].req_rank = -1; - - start_connection_listener_thread (); - start_index_thread (); - return 0; -} - -static int -server_req_queue_destroy (bgpdcmf_request_queue_handle * q) -{ - return 0; -} - -void -dump_packet (char *buf) -{ - - char * - pkt = buf; - int - i; - for (i = 0; i < 1200; i++) - fprintf (stderr, "%c", pkt[i]); - fprintf (stderr, "\n"); -} - -static NNTI_result_t -insert_buf_bufhash (NNTI_buffer_t * buf) -{ - NNTI_result_t - rc = NNTI_OK; - uint32_t - h = - hash6432shift ((uint64_t) buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf. - buf); - - nthread_lock (&nnti_buf_bufhash_lock); - assert (buffers_by_bufhash.find (h) == buffers_by_bufhash.end ()); - buffers_by_bufhash[h] = buf; - nthread_unlock (&nnti_buf_bufhash_lock); - - //log_debug (nnti_debug_level, "bufhash buffer added (buf=%p)", buf); - - return (rc); -} - -static NNTI_buffer_t * -get_buf_bufhash (const uint32_t bufhash) -{ - NNTI_buffer_t * - buf = NULL; -/* - log_debug (nnti_debug_level, "looking for bufhash=%llu", - (uint64_t) bufhash); -*/ - nthread_lock (&nnti_buf_bufhash_lock); - if (buffers_by_bufhash.find (bufhash) != buffers_by_bufhash.end ()) - { - buf = buffers_by_bufhash[bufhash]; - } - nthread_unlock (&nnti_buf_bufhash_lock); - - if (buf != NULL) - { - //log_debug (nnti_debug_level, "buffer found (buf=%p)", buf); - return buf; - } - return (NULL); -} - -static NNTI_buffer_t * -del_buf_bufhash (NNTI_buffer_t * buf) -{ - uint32_t - h = - hash6432shift ((uint64_t) buf->buffer_addr.NNTI_remote_addr_t_u.bgpdcmf. - buf); - log_level - debug_level = nnti_debug_level; - - nthread_lock (&nnti_buf_bufhash_lock); - if (buffers_by_bufhash.find (h) != buffers_by_bufhash.end ()) - { - buf = buffers_by_bufhash[h]; - } - nthread_unlock (&nnti_buf_bufhash_lock); - - if (buf != NULL) - { - log_debug (debug_level, "buffer found"); - buffers_by_bufhash.erase (h); - } - else - { - log_debug (debug_level, "buffer NOT found"); - } - - return (buf); -} - -static void -print_bufhash_map () -{ - if (!logging_debug (nnti_debug_level)) - { - return; - } - - buf_by_bufhash_iter_t - i; - for (i = buffers_by_bufhash.begin (); i != buffers_by_bufhash.end (); i++) - { - log_debug (nnti_debug_level, "bufhash_map key=%llu buf=%p", i->first, - i->second); - } -} - -static NNTI_result_t -insert_wr_wrhash (dcmf_work_request * wr) -{ - NNTI_result_t - rc = NNTI_OK; - uint32_t - h = hash6432shift ((uint64_t) wr); - - nthread_lock (&nnti_wr_wrhash_lock); - assert (wr_by_wrhash.find (h) == wr_by_wrhash.end ()); - wr_by_wrhash[h] = wr; - nthread_unlock (&nnti_wr_wrhash_lock); - - log_debug (nnti_debug_level, - "wrhash work request added (wr=%p ; wr.hash=%llu)", wr, - (uint64_t) h); - - return (rc); -} - -static dcmf_work_request * -get_wr_wrhash (const uint32_t wrhash) -{ - dcmf_work_request * - wr = NULL; - - //log_debug (nnti_debug_level, "looking for wrhash=%llu", (uint64_t) wrhash); - nthread_lock (&nnti_wr_wrhash_lock); - if (wr_by_wrhash.find (wrhash) != wr_by_wrhash.end ()) - { - wr = wr_by_wrhash[wrhash]; - } - nthread_unlock (&nnti_wr_wrhash_lock); - - if (wr != NULL) - { - log_debug (nnti_debug_level, "work request found (wr=%p)", wr); - return wr; - } - - log_debug (nnti_debug_level, "work request NOT found"); - print_wrhash_map (); - - return (NULL); -} - -static dcmf_work_request * -del_wr_wrhash (dcmf_work_request * wr) -{ - uint32_t - h = hash6432shift ((uint64_t) wr); - log_level - debug_level = nnti_debug_level; - - nthread_lock (&nnti_wr_wrhash_lock); - if (wr_by_wrhash.find (h) != wr_by_wrhash.end ()) - { - wr = wr_by_wrhash[h]; - } - nthread_unlock (&nnti_wr_wrhash_lock); - - if (wr != NULL) - { - //log_debug (debug_level, "work request found wr=%p", wr); - wr_by_wrhash.erase (h); - } - else - { - log_debug (debug_level, "work request NOT found"); - } - - return (wr); -} - -static void -print_wrhash_map () -{ - if (!logging_debug (nnti_debug_level)) - { - return; - } - - wr_by_wrhash_iter_t - i; - for (i = wr_by_wrhash.begin (); i != wr_by_wrhash.end (); i++) - { - log_debug (nnti_debug_level, "wrhash_map key=%llu wr=%p", i->first, - i->second); - } -} - - -static int8_t -is_any_buf_op_complete (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, uint32_t * which) -{ - int8_t - rc = FALSE; - - log_debug (nnti_debug_level, "enter"); - - for (uint32_t i = 0; i < buf_count; i++) - { - if ((buf_list[i] != NULL) && - (is_wr_queue_empty (buf_list[i]) == FALSE) && - (is_buf_op_complete (buf_list[i]) == TRUE)) - { - - *which = i; - rc = TRUE; - break; - } - } - - log_debug (nnti_debug_level, "exit (rc=%d)", rc); - - return (rc); -} - -static int8_t -is_wr_queue_empty (const NNTI_buffer_t * reg_buf) -{ - int8_t - rc = FALSE; - bgpdcmf_memory_handle * - dcmf_mem_hdl = NULL; - - log_debug (nnti_debug_level, "enter"); - - dcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - assert (dcmf_mem_hdl); - - if (dcmf_mem_hdl->wr_queue.empty ()) - { - rc = TRUE; - } - - log_debug (nnti_debug_level, "exit (rc=%d)", rc); - return (rc); -} - - -static int8_t -is_buf_op_complete (const NNTI_buffer_t * reg_buf) -{ - int8_t - rc = FALSE; - bgpdcmf_memory_handle * - dcmf_mem_hdl = NULL; - dcmf_work_request * - wr = NULL; - - log_debug (nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - dcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - - if (is_wr_queue_empty (reg_buf) == TRUE) - { - log_debug (nnti_debug_level, - "work request queue is empty - return FALSE"); - rc = FALSE; - } - else - { - wr = dcmf_mem_hdl->wr_queue.front (); - assert (wr); - - rc = is_wr_complete (wr); - } - - if (rc == TRUE) - { - log_debug (nnti_debug_level, "op is complete"); - } - log_debug (nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return (rc); -} - -static int8_t -is_wr_complete (dcmf_work_request * wr) -{ - int8_t - rc = FALSE; - bgpdcmf_memory_handle * - dcmf_mem_hdl = NULL; - - dcmf_mem_hdl = (bgpdcmf_memory_handle *) wr->reg_buf->transport_private; - assert (dcmf_mem_hdl); - - switch (dcmf_mem_hdl->type) - { - case SEND_BUFFER: - if (wr->op_state == SEND_COMPLETE) - { - rc = TRUE; - } - break; - case PUT_SRC_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case GET_DST_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case REQUEST_BUFFER: - case RECEIVE_BUFFER: - if (wr->op_state == RECV_COMPLETE) - { - rc = TRUE; - } - break; - case PUT_DST_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case GET_SRC_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case RDMA_TARGET_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case UNKNOWN_BUFFER: - break; - } - - log_debug (nnti_debug_level, "exit (rc=%d)", rc); - return (rc); -} - -static NNTI_result_t -repost_recv_work_request (NNTI_buffer_t * reg_buf, dcmf_work_request * wr) -{ - bgpdcmf_memory_handle * - bgpdcmf_mem_hdl = NULL; - - log_debug (nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - bgpdcmf_mem_hdl = (bgpdcmf_memory_handle *) reg_buf->transport_private; - assert (bgpdcmf_mem_hdl); - - //memset (&wr->wc, 0, sizeof (nnti_bgpdcmf_work_completion)); - - wr->last_op = 0; - wr->wc.op_complete = 0 ; - - //memset (&wr->op_state, 0, sizeof (bgpdcmf_op_state_t)); - //register_wc (wr); - - bgpdcmf_mem_hdl->wr_queue.push_back(wr); - - log_debug (nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return (NNTI_OK); -} - - diff --git a/packages/trios/libraries/nessie/nnti/nnti_dcmf.h b/packages/trios/libraries/nessie/nnti/nnti_dcmf.h deleted file mode 100644 index 2de797531cbf..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_dcmf.h +++ /dev/null @@ -1,144 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_bgpdcmf.h - * - * Created on: Apr 4, 2012 - * Author: thkorde - */ - -#ifndef NNTI_DCMF_H_ -#define NNTI_DCMF_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "Trios_config.h" - -#include "Trios_nnti.h" -#include "nnti_internal.h" - - - - -NNTI_result_t NNTI_bgpdcmf_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_bgpdcmf_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -NNTI_result_t NNTI_bgpdcmf_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_bgpdcmf_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_bgpdcmf_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - const NNTI_peer_t *peer, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_bgpdcmf_unregister_memory ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_bgpdcmf_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl); - -NNTI_result_t NNTI_bgpdcmf_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset); - -NNTI_result_t NNTI_bgpdcmf_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset); - -NNTI_result_t NNTI_bgpdcmf_wait ( - const NNTI_buffer_t *reg_buf, - const NNTI_buf_ops_t remote_op, - const int timeout, - NNTI_status_t *status); - -NNTI_result_t NNTI_bgpdcmf_waitany ( - const NNTI_buffer_t **buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -NNTI_result_t NNTI_bgpdcmf_waitall ( - const NNTI_buffer_t **buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, - NNTI_status_t **status); - -NNTI_result_t NNTI_bgpdcmf_fini ( - const NNTI_transport_t *trans_hdl); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_DCMF_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/nnti_gni.cpp b/packages/trios/libraries/nessie/nnti/nnti_gni.cpp deleted file mode 100644 index 90adddd4f7dd..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_gni.cpp +++ /dev/null @@ -1,7653 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_gni.c - * - * Created on: Jan 13, 2011 - * Author: thkorde - */ - -#include "Trios_config.h" -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -#ifdef HAVE_TRIOS_HPCTOOLKIT -#include -#define SAMPLING_IS_ACTIVE() hpctoolkit_sampling_is_active() -#define SAMPLING_STOP() hpctoolkit_sampling_stop() -#define SAMPLING_START() hpctoolkit_sampling_start() -#else -#define SAMPLING_IS_ACTIVE() 0 -#define SAMPLING_STOP() -#define SAMPLING_START() -#endif - -#include "nnti_gni.h" -#include "nnti_utils.h" - - - -///* if undefined, the ACK message is NOT sent to the RDMA target when -// * the RDMA op is complete. this creates one-side semantics for RDMA -// * ops. in this mode, the target has no idea when the RDMA op is -// * complete and what data was addressed. -// */ -//#undef USE_RDMA_TARGET_ACK -///* if defined, the RDMA initiator will send an ACK message to the RDMA -// * target when the RDMA op is complete. the target process must wait -// * on the target buffer in order to get the ACK. this creates two-side -// * semantics for RDMA ops. in this mode, when the wait returns the -// * the RDMA op is complete and status indicates what data was addressed. -// */ -//#define USE_RDMA_TARGET_ACK - - - -//#undef USE_ANONYMOUS_MEMORY_REGISTRATION -//#define USE_ANONYMOUS_MEMORY_REGISTRATION - -//#undef USE_RDMA_EVENTS -//#define USE_RDMA_EVENTS - -// NSSI -//#define USE_RDMA_TARGET_ACK -//#define USE_ANONYMOUS_MEMORY_REGISTRATION -//#define USE_RDMA_EVENTS -//#undef USE_RDMA_FENCE - -// ORNL -//#undef USE_RDMA_TARGET_ACK -//#define USE_ANONYMOUS_MEMORY_REGISTRATION -//#undef USE_RDMA_EVENTS -//#undef USE_RDMA_FENCE - - -//#define USE_FMA -//#define USE_BTE -//#define USE_CROSSOVER -//#define USE_MIXED - -//#define FMA_BTE_CROSSOVER 4096 - -///* mode 1 - client uses GNI params from ALPS to create a Commumnication Domain and attach to it */ -//#define USE_ALPS_PTAG -///* mode 2 - client uses a mix of GNI params from ALPS and the server (cookie/pTag) to create a Commumnication Domain and attach to it */ -////#undef USE_ALPS_PTAG - - - -#define NIC_ADDR_BITS 22 -#define NIC_ADDR_SHIFT (32-NIC_ADDR_BITS) -#define NIC_ADDR_MASK 0x3FFFFF -#define CPU_NUM_BITS 7 -#define CPU_NUM_SHIFT (NIC_ADDR_SHIFT-CPU_NUM_BITS) -#define CPU_NUM_MASK 0x7F -#define THREAD_NUM_BITS 3 -#define THREAD_NUM_SHIFT (CPU_NUM_SHIFT-THREAD_NUM_BITS) -#define THREAD_NUM_MASK 0x7 - -#define GNI_INSTID(nic_addr, cpu_num, thr_num) (((nic_addr&NIC_ADDR_MASK)<>NIC_ADDR_SHIFT)&NIC_ADDR_MASK) -#define GNI_CPU_NUMBER(inst_id) ((inst_id>>CPU_NUM_SHIFT)&CPU_NUM_MASK) -#define GNI_THREAD_NUMBER(inst_id) (inst_id&THREAD_NUM_MASK) - -#define NNTI_SMSG_TAG_REQUEST 1 -#define NNTI_SMSG_TAG_CREDIT 2 - -enum nnti_gni_pi_ordering_t { - PI_ORDERING_DEFAULT, - PI_ORDERING_STRICT, - PI_ORDERING_RELAXED -}; -enum nnti_gni_rdma_mode_t { - RDMA_FMA, - RDMA_BTE, - RDMA_MIXED, - RDMA_CROSSOVER -}; -typedef struct { - - bool use_alps_ptag; - - bool use_wr_pool; - uint32_t wr_pool_initial_size; - uint32_t wr_pool_max_size; - bool wr_pool_create_if_empty; - - bool use_rdma_target_ack; - bool use_rdma_events; - bool use_rdma_fence; - - enum nnti_gni_pi_ordering_t pi_ordering; /* DEFAULT, STRICT, RELAXED */ - enum nnti_gni_rdma_mode_t rdma_mode; /* FMA, BTE, MIXED, CROSSOVER */ - - uint32_t fma_bte_crossover_size; - - int32_t max_timeout_ms; - - uint32_t min_atomics_vars; - - uint32_t queue_elements_per_connection; - -} nnti_gni_config_t; - - -/** - * These states are used to signal events between the completion handler - * and the main client or server thread. - * - * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, - * and RDMA_WRITE_COMPLETE for each ping. - */ -typedef enum { - IDLE = 1, - CONNECT_REQUEST, - ADDR_RESOLVED, - ROUTE_RESOLVED, - CONNECTED, - DISCONNECTED, - ERROR -} nnti_gni_connection_state_t; - -typedef enum { - SERVER_CONNECTION, - CLIENT_CONNECTION -} nnti_gni_connection_type_t; - - -#define GNI_OP_PUT_INITIATOR 1 -#define GNI_OP_GET_INITIATOR 2 -#define GNI_OP_PUT_TARGET 3 -#define GNI_OP_GET_TARGET 4 -#define GNI_OP_SEND_REQUEST 5 -#define GNI_OP_SEND_BUFFER 6 -#define GNI_OP_NEW_REQUEST 7 -#define GNI_OP_RECEIVE 8 -#define GNI_OP_FETCH_ADD 9 -#define GNI_OP_COMPARE_SWAP 10 - - -typedef enum { - NNTI_GNI_SGE_STATE_RESET, - NNTI_GNI_SGE_STATE_STARTED, - NNTI_GNI_SGE_STATE_RDMA_COMPLETE, - NNTI_GNI_SGE_STATE_WC_COMPLETE, - NNTI_GNI_SGE_STATE_CANCELING, - NNTI_GNI_SGE_STATE_COMPLETE, - NNTI_GNI_SGE_STATE_ERROR -} nnti_gni_sge_state_t; -typedef enum { - NNTI_GNI_WR_STATE_RESET, // this work request is idle - NNTI_GNI_WR_STATE_POSTED, // target is ready to receive data - NNTI_GNI_WR_STATE_STARTED, // initiator has posted the FMA/RDMA descriptor - NNTI_GNI_WR_STATE_ATTACHED, // target has attached this work request to an NNTI_work_request_t - NNTI_GNI_WR_STATE_CANCELING, // NNTI_cancel() called, but not NNTI_wait() - NNTI_GNI_WR_STATE_RDMA_COMPLETE, // RDMA op complete -// NNTI_GNI_WR_STATE_WC_COMPLETE, // work completion transfer complete - NNTI_GNI_WR_STATE_WAIT_COMPLETE, // NNTI_wait() called and completed successfully - NNTI_GNI_WR_STATE_ERROR // something went wrong. check wr->result -} nnti_gni_wr_state_t; - -typedef struct { - gni_cq_handle_t cq_hdl; - gni_ep_handle_t ep_hdl; -} nnti_gni_conn_ep_t; - - -typedef struct { - uint64_t inst_id; -} nnti_gni_credit_msg_t; - -typedef struct { - uint64_t ack_received; - uint64_t inst_id; - uint64_t byte_len; - uint64_t byte_offset; - uint64_t src_index; - uint64_t src_offset; - uint64_t dest_index; - uint64_t dest_offset; - uint16_t op; -} nnti_gni_work_completion_t; - -// forward declaration -struct nnti_gni_req_queue_mbox_t; - -typedef struct { - NNTI_peer_t peer; - char *peer_name; - NNTI_ip_addr peer_addr; - NNTI_tcp_port peer_port; - uint32_t peer_cookie; - uint32_t peer_ptag; - NNTI_instance_id peer_instance; - - alpsAppGni_t peer_alps_info; - - nnti_gni_req_queue_mbox_t *send_mbox; - nnti_gni_req_queue_mbox_t *recv_mbox; - nthread_counter_t reqs_received; - - uint64_t atomics_addr; - gni_mem_handle_t atomics_mem_hdl; - - gni_cdm_handle_t cdm_hdl; /* a client creates this comm domain with params from the server */ - gni_nic_handle_t nic_hdl; - gni_ep_handle_t ep_hdl; - - nnti_gni_connection_state_t state; - - nnti_gni_connection_type_t connection_type; -} nnti_gni_connection_t; - - -// forward declaration -struct nnti_gni_work_request_t; - -typedef struct { - nnti_gni_work_request_t *gni_wr; - gni_post_descriptor_t post_desc; - nnti_gni_sge_state_t state; -} nnti_gni_sge_t; /* scatter-gather element */ - -typedef struct nnti_gni_work_request_t { - NNTI_work_request_t *nnti_wr; - - nnti_gni_wr_state_t state; - - uint8_t is_initiator; - - nthread_lock_t lock; - - const NNTI_buffer_t *reg_buf; - - nnti_gni_sge_t sge; - nnti_gni_sge_t *sge_list; - uint32_t sge_count; - - uint64_t index; - - nnti_gni_work_completion_t rdma_wc; /* RDMA buffers don't have a work completion footer. use this work for RDMA ops. */ - nnti_gni_work_completion_t *wc; - - uint8_t last_op; - - NNTI_instance_id peer_instance; -} nnti_gni_work_request_t; - -typedef std::deque wr_queue_t; -typedef std::deque::iterator wr_queue_iter_t; - -typedef struct { - gni_mem_handle_t mem_hdl; - gni_mem_handle_t *mem_hdl_list; - uint32_t mem_hdl_count; - wr_queue_t *wr_queue; - nthread_lock_t wr_queue_lock; - uint32_t ref_count; - uint64_t extra; -} nnti_gni_memory_handle_t; - -typedef struct nnti_gni_req_queue_mbox_t { - NNTI_buffer_t *reg_buf; - - nnti_gni_connection_t *conn; - - gni_ep_handle_t ep_hdl; - gni_smsg_attr_t mbox_local_attrs; - gni_smsg_attr_t mbox_remote_attrs; -} nnti_gni_req_queue_mbox_t; - -typedef struct { - NNTI_buffer_t *reg_buf; - - nthread_counter_t req_index; /* index of the next available slot in the request queue */ - - char *req_buffer; /* pointer to the head of the request buffer */ - uint64_t req_size; /* the maximum size of a request */ - int64_t req_count; /* the number of requests that will fit in the queue */ - uint64_t req_buffer_size; /* the size of the request buffer in bytes (req_size*req_count) */ - - int64_t head; /* index of the next work request to assign */ - int64_t tail; /* index of lowest assigned work request */ - - unsigned int bytes_per_mbox; -} nnti_gni_request_queue_handle_t; - -typedef struct { - NNTI_peer_t me; - - uint16_t delivery_mode; - - gni_cdm_handle_t cdm_hdl; - gni_nic_handle_t nic_hdl; - NNTI_instance_id instance; - - gni_cq_handle_t req_send_ep_cq_hdl; /* events are delivered here when a request has been delivered to a remote mailbox */ - gni_cq_handle_t req_send_mem_cq_hdl; /* events are delivered here when an unsolicited credit message is received */ - gni_cq_handle_t req_recv_ep_cq_hdl; - gni_cq_handle_t req_recv_mem_cq_hdl; /* events are delivered here when a request is delivered to a local mailbox */ - - gni_cq_handle_t ep_cq_hdl; - gni_cq_handle_t mem_cq_hdl; - - char interrupt_buf; - gni_mem_handle_t interrupt_mem_hdl; - gni_cq_handle_t interrupt_mem_cq_hdl; - gni_ep_handle_t interrupt_ep_hdl; - gni_cq_handle_t interrupt_ep_cq_hdl; - - uint64_t apid; - alpsAppGni_t alps_info; - - int listen_sock; - char listen_name[NNTI_HOSTNAME_LEN]; - uint32_t listen_addr; /* in NBO */ - uint16_t listen_port; /* in NBO */ - - int64_t *atomics; - gni_mem_handle_t atomics_mem_hdl; - nthread_lock_t atomics_lock; - - nnti_gni_request_queue_handle_t req_queue; -} nnti_gni_transport_global_t; - -typedef struct { - nnti_gni_connection_t *conn; /* the connection where the request arrived */ - uint64_t queue_index; /* the receive queue index where the request will go */ - void *mbox_copy_from; /* the memory address where the request is waiting */ -} nnti_gni_mbox_backlog_t; - - - -static nthread_lock_t nnti_gni_lock; -static nthread_lock_t nnti_mem_lock; -static nthread_lock_t nnti_resend_lock; -static nthread_lock_t nnti_progress_lock; -static nthread_cond_t nnti_progress_cond; - - -static void *aligned_malloc( - size_t size); -static NNTI_result_t setup_atomics(void); -static gni_mem_handle_t register_memory_segment( - NNTI_buf_ops_t ops, - void *buf, - uint64_t len, - uint64_t extra); -static NNTI_result_t register_memory( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t extra, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); -static NNTI_result_t unregister_memory( - gni_mem_handle_t mem_hdl); -static nnti_gni_sge_t *decode_sge( - gni_cq_entry_t *ev_data, - gni_cq_handle_t cq_hdl); -static int cancel_wr( - nnti_gni_work_request_t *gni_wr); -static int process_event( - const NNTI_buffer_t *reg_buf, - nnti_gni_sge_t *gni_sge, - void *header, - gni_cq_handle_t cq_hdl, - gni_cq_entry_t *ev_data, - gni_post_descriptor_t *post_desc_ptr); -static NNTI_result_t execute_mbox_backlog(void); -static NNTI_result_t post_recv_queue_work_requests( - NNTI_buffer_t *reg_buf); -static NNTI_result_t post_recv_work_request( - NNTI_buffer_t *reg_buf); -static NNTI_result_t repost_recv_work_request( - NNTI_buffer_t *reg_buf, - nnti_gni_work_request_t *wr); -static int8_t is_wr_canceling( - nnti_gni_work_request_t *gni_wr); -static int8_t is_wr_complete( - nnti_gni_work_request_t *wr); -static int8_t is_any_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - uint32_t *which_wr); -static int8_t is_all_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); -static void create_status( - NNTI_work_request_t *wr, - nnti_gni_work_request_t *gni_wr, - NNTI_result_t nnti_rc, - gni_cq_entry_t *ev_data, - NNTI_status_t *status); -static void create_peer(NNTI_peer_t *peer, - char *name, - NNTI_ip_addr addr, - NNTI_tcp_port port, - uint32_t ptag, - uint32_t cookie, - NNTI_instance_id instance); -static void copy_peer(NNTI_peer_t *dst_peer, NNTI_peer_t *src_peer); -static int init_server_listen_socket(void); -static int check_listen_socket_for_new_connections(void); -static uint32_t get_cpunum(void); -static void get_alps_info(alpsAppGni_t *alps_info); -static int tcp_read(int sock, void *incoming, size_t len); -static int tcp_write(int sock, const void *outgoing, size_t len); -static int tcp_exchange(int sock, int is_server, void *incoming, void *outgoing, size_t len); -static void transition_connection_to_ready( - int sock, - int is_server, - nnti_gni_connection_t *conn); -static NNTI_result_t get_ipaddr( - char *ipaddr, - int maxlen); -static int create_loopback( - nnti_gni_connection_t *c); -static NNTI_result_t init_connection( - nnti_gni_connection_t **conn, - const int sock, - const int is_server); -static void close_connection(nnti_gni_connection_t *c); -static void print_wc( - const nnti_gni_work_completion_t *wc); -static void print_cq_event( - const gni_cq_entry_t *event, - const bool force); -static void print_post_desc( - const gni_post_descriptor_t *post_desc_ptr); -static int need_mem_cq(NNTI_buf_ops_t ops); -static int need_wc_mem_cq(NNTI_buf_ops_t ops); -static void print_gni_conn(nnti_gni_connection_t *c); -static NNTI_result_t insert_conn_peer(const NNTI_peer_t *peer, nnti_gni_connection_t *conn); -static NNTI_result_t insert_conn_instance(const NNTI_instance_id instance, nnti_gni_connection_t *conn); -static nnti_gni_connection_t *get_conn_peer(const NNTI_peer_t *peer); -static nnti_gni_connection_t *get_conn_instance(const NNTI_instance_id instance); -static NNTI_peer_t *get_peer_by_url(const char *url); -static nnti_gni_connection_t *del_conn_peer(const NNTI_peer_t *peer); -static nnti_gni_connection_t *del_conn_instance(const NNTI_instance_id instance); -static void print_peer_map(void); -static void print_instance_map(void); -static void close_all_conn(void); -//static void print_put_buf(void *buf, uint32_t size); -static void print_raw_buf(void *buf, uint32_t size); - -static NNTI_result_t insert_buf_bufhash(NNTI_buffer_t *buf); -static NNTI_buffer_t *get_buf_bufhash(const uint32_t bufhash); -static NNTI_buffer_t *del_buf_bufhash(NNTI_buffer_t *victim); -static void print_bufhash_map(void); - -static NNTI_result_t insert_sge_sgehash(nnti_gni_sge_t *); -static nnti_gni_sge_t *get_sge_sgehash(const uint32_t bufhash); -static nnti_gni_sge_t *del_sge_sgehash(nnti_gni_sge_t *victim); -static void print_sgehash_map(void); - -static NNTI_result_t wr_pool_init(void); -static nnti_gni_work_request_t *wr_pool_target_pop(void); -static nnti_gni_work_request_t *wr_pool_initiator_pop(void); -static void wr_pool_target_push(nnti_gni_work_request_t *wr); -static void wr_pool_initiator_push(nnti_gni_work_request_t *wr); -static NNTI_result_t wr_pool_fini(void); - -static uint16_t get_dlvr_mode_from_env(); -static void set_dlvr_mode( - gni_post_descriptor_t *pd); -static void set_rdma_mode( - gni_post_descriptor_t *pd); -static void set_post_desc( - gni_post_descriptor_t *pd, - uint8_t op, - uint32_t buf_length); - -static void server_req_queue_init( - nnti_gni_request_queue_handle_t *q, - char *buffer, - uint64_t req_size, - uint64_t extra, - uint64_t req_count); -static void server_req_queue_add_client( - nnti_gni_connection_t *c); -static void server_req_queue_remove_client( - nnti_gni_connection_t *c); -static void server_req_queue_destroy( - nnti_gni_request_queue_handle_t *q); - -static void client_req_queue_init( - nnti_gni_connection_t *c); -static void client_req_queue_destroy( - nnti_gni_connection_t *c); - -static int send_back_credits( - nnti_gni_connection_t *conn, - uint64_t credits_to_send); -static int resend_all(void); -static int request_send( - const NNTI_peer_t *peer_hdl, - nnti_gni_connection_t *conn, - const NNTI_buffer_t *reg_buf, - int req_num, - nnti_gni_work_request_t *gni_wr); -static int send_buffer( - const NNTI_peer_t *peer_hdl, - nnti_gni_connection_t *conn, - const NNTI_buffer_t *src_hdl, - const NNTI_buffer_t *dest_hdl, - nnti_gni_work_request_t *gni_wr); -static int buffer_send( - const NNTI_peer_t *peer_hdl, - nnti_gni_connection_t *conn, - const NNTI_buffer_t *src_hdl, - const NNTI_buffer_t *dest_hdl, - nnti_gni_work_request_t *gni_wr); - -static NNTI_result_t progress( - int timeout, - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -static void config_init( - nnti_gni_config_t *c); -static void config_get_from_env( - nnti_gni_config_t *c); - -inline int DEQUEUE_POST_DESCRIPTOR( - gni_cq_handle_t cq_hdl, - gni_cq_entry_t *ev_data, - gni_post_descriptor_t **post_desc_ptr); - - -#define GNI_MEM_HDL(_b) ((nnti_gni_memory_handle_t *)((_b)->transport_private)) -#define GNI_WORK_REQUEST(_gwr) ((nnti_gni_work_request_t *)((_gwr)->transport_private)) - -#define GNI_ELEMENT_OFFSET(_b,_i) ((_b->payload_size+sizeof(nnti_gni_work_completion_t))*_i) -#define GNI_ELEMENT_ADDRESS(_b,_i) (_b->payload+GNI_ELEMENT_OFFSET(_b,_i)) -#define GNI_WC_ADDRESS(_b,_i) ((nnti_gni_work_completion_t *)(_b->payload + \ - ((_b->payload_size+sizeof(nnti_gni_work_completion_t))*_i) + \ - _b->payload_size)) - -#define GNI_ATTACH_WR(_nwr,_gwr) { \ - _nwr->transport_private=(uint64_t)_gwr; \ - _gwr->nnti_wr =_nwr; \ - } -#define GNI_DETACH_WR(_nwr,_gwr) { \ - log_debug(nnti_debug_level, "_nwr=%p ; _gwr=%p", _nwr, _gwr); \ - _gwr->nnti_wr =NULL; \ - _nwr->transport_private=0; \ - } - -static bool gni_initialized=false; - -static nnti_gni_transport_global_t transport_global_data; -static const int MIN_TIMEOUT = 100; /* in milliseconds. must be >0 for CqVectorMonitor(). */ - -static log_level nnti_cq_debug_level; -static log_level nnti_event_debug_level; -static log_level nnti_ee_debug_level; - - - -/** - * This custom key is used to look up existing connections. - */ -struct addrport_key { - NNTI_ip_addr addr; /* part1 of a compound key */ - NNTI_tcp_port port; /* part2 of a compound key */ - - // Need these operators for the hash map - bool operator<(const addrport_key &key1) const { - if (addr < key1.addr) { - return true; - } else if (addr == key1.addr) { - if (port < key1.port) { - return true; - } - } - return false; - } - bool operator>(const addrport_key &key1) const { - if (addr > key1.addr) { - return true; - } else if (addr == key1.addr) { - if (port > key1.port) { - return true; - } - } - return false; - } - -}; - - -/* Thomas Wang's 64 bit to 32 bit Hash Function (http://www.concentric.net/~ttwang/tech/inthash.htm) */ -static uint32_t hash6432shift(uint64_t key) -{ - key = (~key) + (key << 18); // key = (key << 18) - key - 1; - key = key ^ (key >> 31); - key = key * 21; // key = (key + (key << 2)) + (key << 4); - key = key ^ (key >> 11); - key = key + (key << 6); - key = key ^ (key >> 22); - return (uint32_t)key; -} - -/* - * We need a couple of maps to keep track of connections. Servers need to find - * connections by QP number when requests arrive. Clients need to find connections - * by peer address and port. Setup those maps here. - */ -static std::map connections_by_peer; -typedef std::map::iterator conn_by_peer_iter_t; -typedef std::pair conn_by_peer_t; -static nthread_lock_t nnti_conn_peer_lock; - -static std::map connections_by_instance; -typedef std::map::iterator conn_by_inst_iter_t; -typedef std::pair conn_by_inst_t; -static nthread_lock_t nnti_conn_instance_lock; - -static std::map buffers_by_bufhash; -typedef std::map::iterator buf_by_bufhash_iter_t; -typedef std::pair buf_by_bufhash_t; -static nthread_lock_t nnti_buf_bufhash_lock; - -static std::map sge_by_sgehash; -typedef std::map::iterator sge_by_sgehash_iter_t; -typedef std::pair sge_by_sgehash_t; -static nthread_lock_t nnti_sge_sgehash_lock; - -typedef std::deque wr_pool_t; -typedef std::deque::iterator wr_pool_iter_t; -static nthread_lock_t nnti_wr_pool_lock; - -typedef std::map wr_queue_map_t; -typedef std::map::iterator wr_queue_map_iter_t; - -typedef std::deque mbox_backlog_t; -typedef std::deque::iterator mbox_backlog_iter_t; -static nthread_lock_t nnti_mbox_backlog_lock; - -static wr_pool_t target_wr_pool; -static wr_pool_t initiator_wr_pool; - -static wr_queue_map_t wr_resend_map; -static mbox_backlog_t mbox_backlog; - -static nnti_gni_config_t config; - - -/** - * @brief Initialize NNTI to use a specific transport. - * - * Enable the use of a particular transport by this process. my_url - * allows the process to have some control (if possible) over the - * URL assigned for the transport. For example, a Portals URL to put - * might be "ptl://-1,128". This would tell Portals to use the default - * network ID, but use PID=128. If the transport - * can be initialized without this info (eg. a Portals client), my_url can - * be NULL or empty. - */ -NNTI_result_t NNTI_gni_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl) -{ - int rc=NNTI_OK; - - trios_declare_timer(call_time); - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; - char *sep; - - char hostname[NNTI_HOSTNAME_LEN]; - - uint32_t nic_addr =0; - uint32_t cpu_num =0; - uint32_t thread_num=0; - uint32_t gni_cpu_id=0; - - - assert(trans_hdl); - - log_debug(nnti_ee_debug_level, "enter"); - - log_debug(nnti_debug_level, "my_url=%s", my_url); - log_debug(nnti_debug_level, "initialized=%d, FALSE==%d", (int)gni_initialized, (int)FALSE); - - if (!gni_initialized) { - - memset(&transport_global_data, 0, sizeof(nnti_gni_transport_global_t)); - - nnti_cq_debug_level=nnti_debug_level; - nnti_event_debug_level=nnti_debug_level; - nnti_ee_debug_level=nnti_debug_level; - - nthread_lock_init(&nnti_gni_lock); - nthread_lock_init(&nnti_resend_lock); - nthread_lock_init(&nnti_mbox_backlog_lock); - nthread_lock_init(&nnti_mem_lock); - - nthread_lock_init(&nnti_progress_lock); - nthread_cond_init(&nnti_progress_cond); - - // initialize the mutexes for the connection maps - nthread_lock_init(&nnti_conn_peer_lock); - nthread_lock_init(&nnti_conn_instance_lock); - nthread_lock_init(&nnti_buf_bufhash_lock); - nthread_lock_init(&nnti_sge_sgehash_lock); - - nthread_lock_init(&nnti_wr_pool_lock); - - nthread_lock_init(&transport_global_data.atomics_lock); - - config_init(&config); - config_get_from_env(&config); - - log_debug(nnti_debug_level, "my_url=%s", my_url); - - hostname[0]='\0'; - if (my_url != NULL) { - if ((rc=nnti_url_get_transport(my_url, transport, NNTI_URL_LEN)) != NNTI_OK) { - goto cleanup; - } - if (0!=strcmp(transport, "gni")) { - rc=NNTI_EINVAL; - goto cleanup; - } - - if ((rc=nnti_url_get_address(my_url, address, NNTI_URL_LEN)) != NNTI_OK) { - goto cleanup; - } - - sep=strchr(address, ':'); - if (sep == address) { - /* no hostname given; try gethostname */ - gethostname(hostname, NNTI_HOSTNAME_LEN); - } else { - strncpy(hostname, address, sep-address); - hostname[sep-address]='\0'; - } - sep++; - } else { - rc=get_ipaddr(hostname, NNTI_HOSTNAME_LEN); - if (rc != NNTI_OK) { - log_error(nnti_debug_level, "could not find IP address to listen on"); - goto cleanup; - } - } - strcpy(transport_global_data.listen_name, hostname); - - - transport_global_data.delivery_mode = get_dlvr_mode_from_env(); - - - log_debug(nnti_debug_level, "initializing Gemini"); - -// /* register trace groups (let someone else enable) */ -// trace_counter_gid = trace_get_gid(TRACE_RPC_COUNTER_GNAME); -// trace_interval_gid = trace_get_gid(TRACE_RPC_INTERVAL_GNAME); - - - trios_start_timer(call_time); - get_alps_info(&transport_global_data.alps_info); - trios_stop_timer("get_alps_info", call_time); - - trios_start_timer(call_time); - rc=GNI_CdmGetNicAddress (transport_global_data.alps_info.device_id, &nic_addr, &gni_cpu_id); - trios_stop_timer("CdmGetNicAddress", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CdmGetNicAddress() failed: %d", rc); - if (rc==GNI_RC_NO_MATCH) - rc=NNTI_EEXIST; - else - rc=NNTI_EINVAL; - - goto cleanup; - } - - trios_start_timer(call_time); - cpu_num = get_cpunum(); - trios_stop_timer("get_cpunum", call_time); - - transport_global_data.instance=GNI_INSTID(nic_addr, cpu_num, thread_num); - log_debug(nnti_debug_level, "nic_addr(%llu), cpu_num(%llu), thread_num(%llu), inst_id(%llu), " - "derived.nic_addr(%llu), derived.cpu_num(%llu), derived.thread_num(%llu)", - (uint64_t)nic_addr, (uint64_t)cpu_num, (uint64_t)thread_num, - (uint64_t)transport_global_data.instance, - (uint64_t)GNI_NIC_ADDRESS(transport_global_data.instance), - (uint64_t)GNI_CPU_NUMBER(transport_global_data.instance), - (uint64_t)GNI_THREAD_NUMBER(transport_global_data.instance)); - - log_debug(nnti_debug_level, "global_nic_hdl - host(%s) device_id(%llu) local_addr(%lld) cookie(%llu) ptag(%llu) " - "apid(%llu) inst_id(%llu) gni_nic_addr(%llu) gni_cpu_id(%llu) linux_cpu_num(%llu) omp_thread_num(%llu)", - transport_global_data.listen_name, - (unsigned long long)transport_global_data.alps_info.device_id, - (long long)transport_global_data.alps_info.local_addr, - (unsigned long long)transport_global_data.alps_info.cookie, - (unsigned long long)transport_global_data.alps_info.ptag, - (unsigned long long)transport_global_data.apid, - (unsigned long long)transport_global_data.instance, - (unsigned long long)nic_addr, - (unsigned long long)gni_cpu_id, - (unsigned long long)cpu_num, - (unsigned long long)thread_num); - - trios_start_timer(call_time); - rc=GNI_CdmCreate(transport_global_data.instance, - transport_global_data.alps_info.ptag, - transport_global_data.alps_info.cookie, - GNI_CDM_MODE_ERR_NO_KILL, -// GNI_CDM_MODE_ERR_NO_KILL|GNI_CDM_MODE_BTE_SINGLE_CHANNEL, - &transport_global_data.cdm_hdl); - trios_stop_timer("CdmCreate", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CdmCreate() failed: %d", rc); - rc=NNTI_EINVAL; - goto cleanup; - } - - trios_start_timer(call_time); - - rc=GNI_CdmAttach (transport_global_data.cdm_hdl, - transport_global_data.alps_info.device_id, - (uint32_t*)&transport_global_data.alps_info.local_addr, /* ALPS and GNI disagree about the type of local_addr. cast here. */ - &transport_global_data.nic_hdl); - - trios_stop_timer("CdmAttach", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CdmAttach() failed: %d", rc); - if (rc==GNI_RC_PERMISSION_ERROR) - rc=NNTI_EPERM; - else - rc=NNTI_EINVAL; - - goto cleanup; - } - - rc=GNI_CqCreate ( - transport_global_data.nic_hdl, - 10000, - 0, - GNI_CQ_BLOCKING, - NULL, - NULL, - &transport_global_data.req_send_ep_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.req_send_ep_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_CqCreate ( - transport_global_data.nic_hdl, - 10000, - 0, - GNI_CQ_BLOCKING, - NULL, - NULL, - &transport_global_data.req_send_mem_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.req_send_mem_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_CqCreate ( - transport_global_data.nic_hdl, - 10000, - 0, - GNI_CQ_BLOCKING, - NULL, - NULL, - &transport_global_data.req_recv_ep_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.req_recv_ep_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_CqCreate ( - transport_global_data.nic_hdl, - 10000, - 0, - GNI_CQ_BLOCKING, - NULL, - NULL, - &transport_global_data.req_recv_mem_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.req_recv_mem_cq_hdl) failed: %d", rc); - goto cleanup; - } - - rc=GNI_CqCreate (transport_global_data.nic_hdl, 10000, 0, GNI_CQ_BLOCKING, NULL, NULL, &transport_global_data.ep_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.ep_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_CqCreate (transport_global_data.nic_hdl, 10000, 0, GNI_CQ_BLOCKING, NULL, NULL, &transport_global_data.mem_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.mem_cq_hdl) failed: %d", rc); - goto cleanup; - } - - rc=GNI_CqCreate (transport_global_data.nic_hdl, 10, 0, GNI_CQ_BLOCKING, NULL, NULL, &transport_global_data.interrupt_mem_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.interrupt_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_MemRegister (transport_global_data.nic_hdl, - (uint64_t)&transport_global_data.interrupt_buf, - sizeof(uint32_t), - transport_global_data.interrupt_mem_cq_hdl, - GNI_MEM_READWRITE, - (uint32_t)-1, - &transport_global_data.interrupt_mem_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "MemRegister(transport_global_data.interrupt_mem_hdl) failed: rc=%d, %s", rc, strerror(errno)); - goto cleanup; - } - - rc=GNI_CqCreate (transport_global_data.nic_hdl, 1, 0, GNI_CQ_BLOCKING, NULL, NULL, &transport_global_data.interrupt_ep_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.interrupt_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_EpCreate (transport_global_data.nic_hdl, transport_global_data.interrupt_ep_cq_hdl, &transport_global_data.interrupt_ep_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpCreate(transport_global_data.interrupt_ep_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_EpBind (transport_global_data.interrupt_ep_hdl, transport_global_data.alps_info.local_addr, transport_global_data.instance); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpBind(transport_global_data.interrupt_ep_hdl) failed: %d", rc); - goto cleanup; - } - - if (config.use_wr_pool) { - rc=wr_pool_init(); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "wr_pool_init(): %d", rc); - goto cleanup; - } - } - - setup_atomics(); - - trios_start_timer(call_time); - init_server_listen_socket(); - trios_stop_timer("init_server_listen_socket", call_time); - - if (logging_info(nnti_debug_level)) { - fprintf(logger_get_file(), "Gemini Initialized: host(%s) port(%u)\n", - transport_global_data.listen_name, - ntohs(transport_global_data.listen_port)); - } - - create_peer( - &transport_global_data.me, - transport_global_data.listen_name, - transport_global_data.listen_addr, - transport_global_data.listen_port, - transport_global_data.alps_info.ptag, - transport_global_data.alps_info.cookie, - transport_global_data.instance); - copy_peer( - &trans_hdl->me, - &transport_global_data.me); - - gni_initialized = true; - } - -cleanup: - log_debug(nnti_ee_debug_level, "exit"); - - return((NNTI_result_t)rc); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recognize. - * - * URL format: "transport://address/memory_descriptor" - * - transport - (required) identifies how the URL should parsed - * - address - (required) uniquely identifies a location on the network - * - ex. "ptl://nid:pid/", "gni://ip_addr:port" - * - memory_descriptor - (optional) transport-specific representation of RMA params - */ -NNTI_result_t NNTI_gni_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen) -{ - NNTI_result_t rc=NNTI_OK; - - assert(trans_hdl); - assert(url); - assert(maxlen>0); - - log_debug(nnti_ee_debug_level, "enter"); - - strncpy(url, trans_hdl->me.url, maxlen); - url[maxlen-1]='\0'; - - log_debug(nnti_ee_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * - * - * Connectionless transport: parse and populate - * Connected transport: parse, connection and populate - * - */ -NNTI_result_t NNTI_gni_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl) -{ - int rc=NNTI_OK; - - trios_declare_timer(call_time); - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; - char params[NNTI_URL_LEN]; - char *sep; - - NNTI_peer_t *existing_peer=NULL; - - char hostname[NNTI_HOSTNAME_LEN]; - char port_str[NNTI_HOSTNAME_LEN]; - NNTI_tcp_port port; - - char *cookie_str; - char *ptag_str; - - int s; - struct hostent *host_entry; - struct sockaddr_in skin; - socklen_t skin_size=sizeof(struct sockaddr_in); - - nnti_gni_connection_t *conn=NULL; - - long start_time; - long elapsed_time = 0; - long timeout_per_call; - - - assert(trans_hdl); - assert(peer_hdl); - - log_debug(nnti_ee_debug_level, "enter (url=%s)", url); - - existing_peer=get_peer_by_url(url); - if (existing_peer!=NULL) { - *peer_hdl=*existing_peer; - return(NNTI_OK); - } - - conn = (nnti_gni_connection_t *)calloc(1, sizeof(nnti_gni_connection_t)); - log_debug(nnti_debug_level, "calloc returned conn=%p.", conn); - if (conn == NULL) { - log_error(nnti_debug_level, "calloc returned NULL. out of memory?: %s", strerror(errno)); - rc=NNTI_ENOMEM; - goto cleanup; - } - nthread_counter_init(&conn->reqs_received); - - if (url != NULL) { - if ((rc=nnti_url_get_transport(url, transport, NNTI_URL_LEN)) != NNTI_OK) { - goto cleanup; - } - if (0!=strcmp(transport, "gni")) { - /* the peer described by 'url' is not an Gemini peer */ - rc=NNTI_EINVAL; - goto cleanup; - } - - if ((rc=nnti_url_get_address(url, address, NNTI_URL_LEN)) != NNTI_OK) { - /* the peer described by 'url' is not an Gemini peer */ - rc=NNTI_EINVAL; - goto cleanup; - } - - if ((rc=nnti_url_get_params(url, params, NNTI_URL_LEN)) != NNTI_OK) { - /* the peer described by 'url' is not an Gemini peer */ - rc=NNTI_EINVAL; - goto cleanup; - } - - sep=strchr(address, ':'); - strncpy(hostname, address, sep-address); - hostname[sep-address]='\0'; - strcpy(port_str, sep+1); - port=strtol(port_str, NULL, 0); - - log_debug(nnti_ee_debug_level, "params=%s", params); - - ptag_str=strstr(params, "ptag="); - sep=strchr(ptag_str, '&'); - *sep='\0'; - log_debug(nnti_ee_debug_level, "ptag_str=%s", ptag_str+5); - conn->peer_ptag=strtol(ptag_str+5, NULL, 10); - *sep='&'; - - cookie_str=strstr(params, "cookie="); - log_debug(nnti_ee_debug_level, "cookie_str=%s", cookie_str+7); - conn->peer_cookie=strtoull(cookie_str+7, NULL, 10); - - log_debug(nnti_ee_debug_level, "url=%s", url); - - } else { - /* */ - rc=NNTI_EINVAL; - goto cleanup; - } - - if (config.use_alps_ptag) { - conn->cdm_hdl = transport_global_data.cdm_hdl; - conn->nic_hdl = transport_global_data.nic_hdl; - } else { - log_debug(nnti_ee_debug_level, "conn_nic_hdl - host(%s) device_id(%llu) local_addr(%lld) cookie(%llu) ptag(%llu) " - "apid(%llu) inst_id(%llu) gni_nic_addr(%llu) linux_cpu_num(%llu) omp_thread_num(%llu)", - transport_global_data.listen_name, - (unsigned long long)transport_global_data.alps_info.device_id, - (long long)transport_global_data.alps_info.local_addr, - (unsigned long long)conn->peer_cookie, - (unsigned long long)conn->peer_ptag, - (unsigned long long)transport_global_data.apid, - (unsigned long long)transport_global_data.instance, - (uint64_t)GNI_NIC_ADDRESS(transport_global_data.instance), - (uint64_t)GNI_CPU_NUMBER(transport_global_data.instance), - (uint64_t)GNI_THREAD_NUMBER(transport_global_data.instance)); - - trios_start_timer(call_time); - rc=GNI_CdmCreate(transport_global_data.instance, - conn->peer_ptag, - conn->peer_cookie, - GNI_CDM_MODE_ERR_ALL_KILL, - &conn->cdm_hdl); - trios_stop_timer("CdmCreate", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CdmCreate() failed: %d", rc); - rc=NNTI_EINVAL; - goto cleanup; - } - - trios_start_timer(call_time); - rc=GNI_CdmAttach (conn->cdm_hdl, - transport_global_data.alps_info.device_id, - (uint32_t*)&transport_global_data.alps_info.local_addr, /* ALPS and GNI disagree about the type of local_addr. cast here. */ - &conn->nic_hdl); - trios_stop_timer("CdmAttach", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CdmAttach() failed: %d", rc); - if (rc==GNI_RC_PERMISSION_ERROR) - rc=NNTI_EPERM; - else - rc=NNTI_EINVAL; - goto cleanup; - } - - rc=GNI_CqCreate (conn->nic_hdl, 10000, 0, GNI_CQ_BLOCKING, NULL, NULL, &transport_global_data.ep_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.ep_cq_hdl) failed: %d", rc); - goto cleanup; - } - rc=GNI_CqCreate (conn->nic_hdl, 10000, 0, GNI_CQ_BLOCKING, NULL, NULL, &transport_global_data.mem_cq_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CqCreate(transport_global_data.mem_cq_hdl) failed: %d", rc); - goto cleanup; - } - } - - host_entry = gethostbyname(hostname); - if (!host_entry) { - log_warn(nnti_debug_level, "failed to resolve server name (%s): %s", hostname, strerror(errno)); - rc=NNTI_ENOENT; - goto cleanup; - } - memset(&skin, 0, sizeof(skin)); - skin.sin_family = host_entry->h_addrtype; - memcpy(&skin.sin_addr, host_entry->h_addr_list[0], (size_t) host_entry->h_length); - skin.sin_port = htons(port); - - if (((uint32_t)skin.sin_addr.s_addr == transport_global_data.listen_addr) && - ((uint32_t)skin.sin_port == transport_global_data.listen_port)) { - - create_loopback(conn); - - create_peer( - peer_hdl, - conn->peer_name, - conn->peer_addr, - conn->peer_port, - conn->peer_ptag, - conn->peer_cookie, - conn->peer_instance); - - conn->peer=*peer_hdl; - - insert_conn_peer(peer_hdl, conn); - insert_conn_instance(conn->peer_instance, conn); - - } else { - elapsed_time=0; - timeout_per_call = MIN_TIMEOUT; - - s = socket(AF_INET, SOCK_STREAM, 0); - if (s < 0) { - log_warn(nnti_debug_level, "failed to create tcp socket: errno=%d (%s)", errno, strerror(errno)); - rc=NNTI_EIO; - goto cleanup; - } - trios_start_timer(call_time); - start_time=trios_get_time_ms(); - while((timeout==-1) || (elapsed_time < timeout)) { - log_debug(nnti_debug_level, "calling connect"); - if (connect(s, (struct sockaddr *)&skin, skin_size) == 0) { - log_debug(nnti_debug_level, "connected"); - break; - } - elapsed_time=trios_get_time_ms()-start_time; - log_warn(nnti_debug_level, "failed to connect to server (%s:%u): errno=%d (%s)", hostname, port, errno, strerror(errno)); - if ((timeout>0) && (elapsed_time >= timeout)) { - rc=NNTI_EIO; - goto cleanup; - } - nnti_sleep(timeout_per_call); - } - trios_stop_timer("socket connect", call_time); - - trios_start_timer(call_time); - rc = init_connection(&conn, s, 0); - trios_stop_timer("gni init connection", call_time); - if (conn==NULL) { - rc=NNTI_EIO; - goto cleanup; - } - - create_peer( - peer_hdl, - conn->peer_name, - conn->peer_addr, - conn->peer_port, - conn->peer_ptag, - conn->peer_cookie, - conn->peer_instance); - - conn->peer=*peer_hdl; - - insert_conn_peer(peer_hdl, conn); - insert_conn_instance(conn->peer_instance, conn); - - transition_connection_to_ready(s, 0, conn); - - if (close(s) < 0) { - log_warn(nnti_debug_level, "failed to close tcp socket: errno=%d (%s)", errno, strerror(errno)); - rc=NNTI_EIO; - goto cleanup; - } - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer_hdl", - "end of NNTI_gni_connect", peer_hdl); - } - -cleanup: - if (rc != NNTI_OK) { - if (conn!=NULL) free(conn); - } - log_debug(nnti_ee_debug_level, "exit"); - - return((NNTI_result_t)rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - */ -NNTI_result_t NNTI_gni_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - assert(trans_hdl); - assert(peer_hdl); - - log_debug(nnti_ee_debug_level, "enter"); - - nnti_gni_connection_t *conn=get_conn_peer(peer_hdl); - - close_connection(conn); - - del_conn_peer(peer_hdl); - del_conn_instance(conn->peer_instance); - - log_debug(nnti_ee_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_gni_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - /* each element needs some extra space to store the work completion data */ - uint32_t transport_extra=sizeof(nnti_gni_work_completion_t); - - assert(trans_hdl); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - char *buf=(char *)aligned_malloc((element_size+transport_extra)*num_elements); - assert(buf); - - nnti_rc=register_memory( - trans_hdl, - buf, - element_size, - transport_extra, - num_elements, - ops, - reg_buf); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_gni_alloc", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_gni_free ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - char *buf=NNTI_BUFFER_C_POINTER(reg_buf); - assert(buf); - - nnti_rc=NNTI_gni_unregister_memory(reg_buf); - - free(buf); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_gni_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - rc=register_memory( - trans_hdl, - buffer, - element_size, - 0, //extra - num_elements, - ops, - reg_buf); - - return(rc); -} - - -/** - * @brief Prepare a list of memory segments for network operations. - * - * Wrap a list of user allocated memory segments in an NNTI_buffer_t. The - * transport may take additional actions to prepare the memory segments for - * network send/receive. If the memory segments don't meet the transport's - * requirements for memory regions, then errors or poor performance may - * result. - * - */ -NNTI_result_t NNTI_gni_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - gni_return_t gni_rc =GNI_RC_SUCCESS; - - NNTI_buffer_t *old_buf=NULL; - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - assert(trans_hdl); - assert(segments); - assert(segment_lengths); - assert(num_segments>0); - assert(ops>0); - assert(reg_buf); - - log_debug(nnti_ee_debug_level, "enter (reg_buf=%p)", reg_buf); - - if (ops == NNTI_BOP_RECV_QUEUE) { - log_debug(nnti_debug_level, "NNTI_BOP_RECV_QUEUE type cannot be segmented."); - return(NNTI_EINVAL); - } - -nthread_lock(&nnti_mem_lock); - - old_buf=get_buf_bufhash(hash6432shift((uint64_t)segments[0])); - if (old_buf==NULL) { - gni_mem_hdl=(nnti_gni_memory_handle_t*)calloc(1, sizeof(nnti_gni_memory_handle_t)); - assert(gni_mem_hdl); - gni_mem_hdl->wr_queue =new wr_queue_t; - gni_mem_hdl->ref_count=1; - nthread_lock_init(&gni_mem_hdl->wr_queue_lock); - } else { - - - // check that the number of old_buf segments equals num_segments. - if (old_buf->buffer_segments.NNTI_remote_addr_array_t_len != num_segments) { - log_debug(nnti_debug_level, "Segment count mismatch (old=%d new=%d). Aborting...", - old_buf->buffer_segments.NNTI_remote_addr_array_t_len, num_segments); - return(NNTI_EINVAL); - } - // check that all the old_buf segments match the current list of segments - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.buf != (uint64_t)segments[i]) { - log_debug(nnti_debug_level, "Segment address mismatch (old[%d]=%p new[%d]=%p).", - i, old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.buf, i, segments[i]); - return(NNTI_EINVAL); - } - if (old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size != segment_lengths[i]) { - log_debug(nnti_debug_level, "Segment length mismatch (old[%d]=%d new[%d]=%d).", - i, old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size, i, segment_lengths[i]); - return(NNTI_EINVAL); - } - } - - gni_mem_hdl=GNI_MEM_HDL(old_buf); - gni_mem_hdl->ref_count++; - } - - log_debug(nnti_ee_debug_level, "gni_mem_hdl->ref_count==%lu", gni_mem_hdl->ref_count); - - memset(reg_buf, 0, sizeof(NNTI_buffer_t)); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size=0; - for (uint32_t i=0;ipayload_size += segment_lengths[i]; - } - reg_buf->payload = (uint64_t)segments[0]; - reg_buf->transport_private = (uint64_t)gni_mem_hdl; - - log_debug(nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - - if (gni_mem_hdl->ref_count==1) { - - gni_mem_hdl->extra=0; - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(num_segments, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=num_segments; - - gni_mem_hdl->mem_hdl_list =(gni_mem_handle_t *)calloc(num_segments, sizeof(gni_mem_handle_t)); - gni_mem_hdl->mem_hdl_count=num_segments; - - for (uint32_t i=0;imem_hdl_list[i]=register_memory_segment( - reg_buf->ops, - segments[i], - segment_lengths[i], - gni_mem_hdl->extra); - - log_debug(nnti_debug_level, "hdl->mem_hdl_list[%d]=(%llu,%llu)", i, (uint64_t)gni_mem_hdl->mem_hdl_list[i].qword1, (uint64_t)gni_mem_hdl->mem_hdl_list[i].qword2); - } - } - - if (config.use_rdma_target_ack) { - if ((ops & NNTI_BOP_REMOTE_READ) || - (ops & NNTI_BOP_REMOTE_WRITE)) { - post_recv_work_request(reg_buf); - } - } - - if (nnti_rc==NNTI_OK) { - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_val[i].transport_id = NNTI_TRANSPORT_GEMINI; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.mem_hdl.qword1 = gni_mem_hdl->mem_hdl_list[i].qword1; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.mem_hdl.qword2 = gni_mem_hdl->mem_hdl_list[i].qword2; - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size = segment_lengths[i]; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.buf = (uint64_t)segments[i]; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.type = NNTI_GNI_SEND_SRC; - } - } - - if (gni_mem_hdl->ref_count==1) { - insert_buf_bufhash(reg_buf); - log_debug(nnti_debug_level, "reg_buf.buf.hash==%llu", - (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - log_debug(nnti_debug_level, "ref_count==1 called insert_buf_bufhash() (reg_buf=%p, reg_buf.hash6432=%llu)", - reg_buf, (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - NNTI_buffer_t *tmp_buf=get_buf_bufhash((uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - log_debug(nnti_debug_level, "immediate get_buf_bufhash() says tmp_buf=%p", tmp_buf); - } - -nthread_unlock(&nnti_mem_lock); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_gni_register_memory", reg_buf); - } - - log_debug(nnti_ee_debug_level, "exit (reg_buf=%p, reg_buf.hash6432=%llu)", reg_buf, (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - - switch(gni_rc) { - case GNI_RC_SUCCESS: - nnti_rc=NNTI_OK; - default: - nnti_rc=NNTI_EIO; - } - - return (nnti_rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_gni_unregister_memory ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - assert(reg_buf); - - log_debug(nnti_ee_debug_level, "enter (reg_buf=%p)", reg_buf); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "start of NNTI_gni_unregister_memory", reg_buf); - } - -nthread_lock(&nnti_mem_lock); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)reg_buf->transport_private; - assert(gni_mem_hdl); - gni_mem_hdl->ref_count--; - - log_debug(nnti_ee_debug_level, "gni_mem_hdl->ref_count==%lu", gni_mem_hdl->ref_count); - - if (gni_mem_hdl->ref_count==0) { - log_debug(nnti_ee_debug_level, "gni_mem_hdl->ref_count is 0. release all resources."); - - if (reg_buf->ops==NNTI_BOP_RECV_QUEUE) { - server_req_queue_destroy( - &transport_global_data.req_queue); - - } else { - for (uint32_t i=0;imem_hdl_count;i++) { - unregister_memory(gni_mem_hdl->mem_hdl_list[i]); - } - if (gni_mem_hdl->mem_hdl_count > 1) { - free(gni_mem_hdl->mem_hdl_list); - } - gni_mem_hdl->mem_hdl_list =NULL; - gni_mem_hdl->mem_hdl_count=0; - } - - del_buf_bufhash(reg_buf); - - nthread_lock(&gni_mem_hdl->wr_queue_lock); - while (!gni_mem_hdl->wr_queue->empty()) { - nnti_gni_work_request_t *wr=gni_mem_hdl->wr_queue->front(); - log_debug(nnti_debug_level, "removing pending wr=%p", wr); - gni_mem_hdl->wr_queue->pop_front(); - if (config.use_wr_pool) { - if (wr->is_initiator==TRUE) { - wr_pool_initiator_push(wr); - } else { - wr_pool_target_push(wr); - } - } else { - log_debug(nnti_debug_level, "free(wr) (reg_buf=%p, wr=%p)", reg_buf, wr); - free(wr); - } - } - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - nthread_lock_fini(&gni_mem_hdl->wr_queue_lock); - - reg_buf->transport_id = NNTI_TRANSPORT_NULL; - GNI_SET_MATCH_ANY(®_buf->buffer_owner); - reg_buf->ops = (NNTI_buf_ops_t)0; - reg_buf->payload_size = 0; - reg_buf->payload = 0; - reg_buf->transport_private = 0; - - if (gni_mem_hdl) { - delete gni_mem_hdl->wr_queue; - free(gni_mem_hdl); - } - if (reg_buf->buffer_segments.NNTI_remote_addr_array_t_val) { - free(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=NULL; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=0; - } - - memset(reg_buf, 0, sizeof(NNTI_buffer_t)); - } - -nthread_unlock(&nnti_mem_lock); - - log_debug(nnti_ee_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - */ -NNTI_result_t NNTI_gni_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - trios_declare_timer(call_time); - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - nnti_gni_work_request_t *gni_wr=NULL; - - log_debug(nnti_ee_debug_level, "enter"); - - assert(peer_hdl); - assert(msg_hdl); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)msg_hdl->transport_private; - assert(gni_mem_hdl); - - if (config.use_wr_pool) { - gni_wr=wr_pool_initiator_pop(); - } else { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - } - assert(gni_wr); - - gni_wr->nnti_wr=wr; - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "msg_hdl", - "NNTI_send", msg_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_hdl", - "NNTI_send", dest_hdl); - } - - wr->transport_id =msg_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)msg_hdl; - wr->ops =msg_hdl->ops; - wr->transport_private=(uint64_t)gni_wr; - - log_debug(nnti_debug_level, "sending to (%s, instance=%llu)", peer_hdl->url, (uint64_t)peer_hdl->peer.NNTI_remote_process_t_u.gni.inst_id); - - if ((dest_hdl == NULL) || (dest_hdl->ops == NNTI_BOP_RECV_QUEUE)) { - nnti_gni_connection_t *conn=get_conn_peer(peer_hdl); - assert(conn); - - trios_start_timer(call_time); - request_send(peer_hdl, conn, msg_hdl, 0, gni_wr); - trios_stop_timer("send to request queue", call_time); - - } else { - nnti_gni_connection_t *conn=get_conn_peer(peer_hdl); - assert(conn); - - trios_start_timer(call_time); - buffer_send(peer_hdl, conn, msg_hdl, dest_hdl, gni_wr); - trios_stop_timer("send to receive dest", call_time); - - wr->result=NNTI_OK; -// NNTI_gni_put(msg_hdl, 0, msg_hdl->payload_size, dest_hdl, 0, wr); - } - - log_debug(nnti_ee_debug_level, "exit (wr=%p ; gni_wr=%p)", wr, gni_wr); - - return(nnti_rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_gni_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - int rc=NNTI_OK; - trios_declare_timer(call_time); - - gni_ep_handle_t ep_hdl; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - nnti_gni_work_request_t *gni_wr=NULL; - - uint64_t remaining=0; - - bool use_fma=true; - - struct { - uint32_t segment_index; // index of the current segment - uint64_t segment_offset; // offset into the current segment - uint32_t segment_remaining; // bytes remaining in the current segment - uint32_t last_segment; // index of the last segment in this transfer - } src_params, dst_params, src_copy, dst_copy; - - uint32_t sge_index=0; - - log_debug(nnti_ee_debug_level, "enter"); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - if (src_offset > src_buffer_hdl->payload_size) { - } - if (src_buffer_hdl->payload_size < src_length-src_offset) { - } - if (dest_offset > dest_buffer_hdl->payload_size) { - } - if (dest_buffer_hdl->payload_size < src_length-dest_offset) { - } - - gni_mem_hdl=(nnti_gni_memory_handle_t *)src_buffer_hdl->transport_private; - assert(gni_mem_hdl); - - if (config.use_wr_pool) { - gni_wr=wr_pool_initiator_pop(); - } else { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - } - assert(gni_wr); - - gni_wr->reg_buf=src_buffer_hdl; - gni_wr->last_op=GNI_OP_PUT_INITIATOR; - gni_wr->peer_instance=dest_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.gni.inst_id; - log_debug(nnti_event_debug_level, "gni_wr->peer_instance==%lu", gni_wr->peer_instance); - - gni_wr->nnti_wr=wr; - - wr->transport_id =src_buffer_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)src_buffer_hdl; - wr->ops =NNTI_BOP_LOCAL_READ; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)gni_wr; - - nnti_gni_connection_t *conn=get_conn_peer(&dest_buffer_hdl->buffer_owner); - assert(conn); - ep_hdl=conn->ep_hdl; - - nthread_lock(&gni_mem_hdl->wr_queue_lock); - gni_mem_hdl->wr_queue->push_back(gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - // the src_offset could exclude some local segments from this operation. find the first segment. - src_params.segment_offset =src_offset; - src_params.segment_remaining=0; - src_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "src_offset=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_buffer.segment[%lu].size==%lu", - src_offset, src_params.segment_offset, src_params.segment_remaining, - i, src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size); - if (src_params.segment_offset > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - src_params.segment_offset -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - - log_debug(nnti_debug_level, "src_params.segment_index=%lu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - continue; - } else { - src_params.segment_index = i; - src_params.segment_remaining = src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size - src_params.segment_offset; - - log_debug(nnti_debug_level, "src_params.segment_index=%lu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=src_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - remaining -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - } else { - src_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "src_params.segment_index=%lu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_params.last_segment=%lu", - src_params.segment_index, src_params.segment_offset, src_params.segment_remaining, src_params.last_segment); - - // the dest_offset could exclude some local segments from this operation. find the first segment. - dst_params.segment_offset =dest_offset; - dst_params.segment_remaining=0; - dst_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "dest_offset=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dest_buffer.segment[%lu].size==%lu", - dest_offset, dst_params.segment_offset, dst_params.segment_remaining, - i, dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size); - if (dst_params.segment_offset > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - dst_params.segment_offset -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - - log_debug(nnti_debug_level, "dst_params.segment_index=%lu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - continue; - } else { - dst_params.segment_index = i; - dst_params.segment_remaining = dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size - dst_params.segment_offset; - - log_debug(nnti_debug_level, "dst_params.segment_index=%lu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=dst_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - remaining -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - } else { - dst_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "dst_params.segment_index=%lu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dst_params.last_segment=%lu", - dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining, dst_params.last_segment); - - /* START calculate the number of SGEs required */ - { - src_copy=src_params; - dst_copy=dst_params; - - NNTI_remote_addr_t *src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_copy.segment_index]; - NNTI_remote_addr_t *dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_copy.segment_index]; - - gni_wr->sge_count=0; - - while (src_copy.segment_index <= src_copy.last_segment) { - - gni_wr->sge_count++; - - if (src_copy.segment_remaining < dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment with extra space in the destination"); - // the remaining source segment fits in the remaining destination segment with extra space - dst_copy.segment_offset += src_copy.segment_remaining; - dst_copy.segment_remaining -= src_copy.segment_remaining; - - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else if (src_copy.segment_remaining == dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment EXACTLY"); - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else { - log_debug(nnti_debug_level, "the remaining source segment DOES NOT fit in the remaining destination segment"); - // the remaining source segment DOES NOT fit in the remaining destination segment - src_copy.segment_offset += dst_copy.segment_remaining; - src_copy.segment_remaining -= dst_copy.segment_remaining; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } - - log_debug(nnti_debug_level, "src_copy.segment_index=%lu, src_copy.segment_offset=%llu, src_copy.segment_remaining=%lu, src_copy.last_segment=%lu", - src_copy.segment_index, src_copy.segment_offset, src_copy.segment_remaining, src_copy.last_segment); - log_debug(nnti_debug_level, "dst_copy.segment_index=%lu, dst_copy.segment_offset=%llu, dst_copy.segment_remaining=%lu, dst_copy.last_segment=%lu", - dst_copy.segment_index, dst_copy.segment_offset, dst_copy.segment_remaining, dst_copy.last_segment); - - } - } - /* END calculate the number of SGEs required */ - - NNTI_remote_addr_t *src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_params.segment_index]; - NNTI_remote_addr_t *dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_params.segment_index]; - - gni_wr->sge_list = (nnti_gni_sge_t *)malloc(gni_wr->sge_count*sizeof(nnti_gni_sge_t)); - - log_debug(nnti_debug_level, "this get requires %d SGEs", gni_wr->sge_count); - - sge_index=0; - - while (src_params.segment_index <= src_params.last_segment) { - - memset(&gni_wr->sge_list[sge_index].post_desc, 0, sizeof(gni_post_descriptor_t)); - - gni_wr->sge_list[sge_index].post_desc.src_cq_hndl=transport_global_data.ep_cq_hdl; - - gni_wr->sge_list[sge_index].post_desc.local_addr =src_remote_addr->NNTI_remote_addr_t_u.gni.buf+src_params.segment_offset; - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword1 =src_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword1; - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword2 =src_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword2; - - gni_wr->sge_list[sge_index].post_desc.remote_addr =dst_remote_addr->NNTI_remote_addr_t_u.gni.buf+dst_params.segment_offset; - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword1=dst_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword1; - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword2=dst_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword2; - - if (src_params.segment_remaining <= dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - gni_wr->sge_list[sge_index].post_desc.length=src_params.segment_remaining; - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - gni_wr->sge_list[sge_index].post_desc.length=dst_params.segment_remaining; - } - - set_post_desc(&gni_wr->sge_list[sge_index].post_desc, GNI_OP_PUT_INITIATOR, gni_wr->sge_list[sge_index].post_desc.length); - - gni_wr->sge_list[sge_index].state=NNTI_GNI_SGE_STATE_STARTED; - - gni_wr->sge_list[sge_index].gni_wr=gni_wr; - - gni_wr->wc=&gni_wr->rdma_wc; - gni_wr->wc->op =GNI_OP_PUT_TARGET; - gni_wr->wc->byte_len =gni_wr->sge_list[sge_index].post_desc.length; - gni_wr->wc->src_index =src_params.segment_index; - gni_wr->wc->src_offset =src_params.segment_offset; - gni_wr->wc->dest_index =dst_params.segment_index; - gni_wr->wc->dest_offset=dst_params.segment_offset; - gni_wr->wc->inst_id =transport_global_data.instance; - - print_wc(gni_wr->wc); - - if (config.rdma_mode==RDMA_CROSSOVER) { - if (gni_wr->sge_list[sge_index].post_desc.length > config.fma_bte_crossover_size) { - use_fma=false; - } else { - use_fma=true; - } - } else if ((config.rdma_mode==RDMA_BTE) || (config.rdma_mode==RDMA_MIXED)) { - use_fma=false; - } else if (config.rdma_mode==RDMA_FMA) { - use_fma=true; - } - - insert_sge_sgehash(&gni_wr->sge_list[sge_index]); - - nthread_lock(&nnti_gni_lock); - GNI_EpSetEventData( - ep_hdl, - hash6432shift((uint64_t)&gni_wr->sge_list[sge_index]), - hash6432shift((uint64_t)dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - - if (use_fma) { - log_debug(nnti_event_debug_level, "calling PostFma(fma put ; ep_hdl(%llu) transport_global_data.ep_cq_hdl(%llu) local_mem_hdl(%llu, %llu) remote_mem_hdl(%llu, %llu))", - ep_hdl, transport_global_data.ep_cq_hdl, - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword2, - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword2); - trios_start_timer(call_time); - rc=GNI_PostFma(ep_hdl, &gni_wr->sge_list[sge_index].post_desc); - trios_stop_timer("PostFma put", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "failed to post FMA (rc=%d): %s", rc, strerror(errno)); - rc=NNTI_EIO; - } - } else { - log_debug(nnti_event_debug_level, "calling PostRdma(rdma put ; ep_hdl(%llu) transport_global_data.ep_cq_hdl(%llu) local_mem_hdl(%llu, %llu) remote_mem_hdl(%llu, %llu))", - ep_hdl, transport_global_data.ep_cq_hdl, - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword2, - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword2); - trios_start_timer(call_time); - rc=GNI_PostRdma(ep_hdl, &gni_wr->sge_list[sge_index].post_desc); - trios_stop_timer("PostRdma put", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "failed to post RDMA (rc=%d): %s", rc, strerror(errno)); - rc=NNTI_EIO; - } - } - nthread_unlock(&nnti_gni_lock); - - if (src_params.segment_remaining < dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - dst_params.segment_offset += src_params.segment_remaining; - dst_params.segment_remaining -= src_params.segment_remaining; - - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else if (src_params.segment_remaining == dst_params.segment_remaining) { - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - src_params.segment_offset += dst_params.segment_remaining; - src_params.segment_remaining -= dst_params.segment_remaining; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } - - sge_index++; - } - - log_debug(nnti_ee_debug_level, "exit (wr=%p ; gni_wr=%p)", wr, gni_wr); - - return((NNTI_result_t)rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_gni_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - int rc=NNTI_OK; - trios_declare_timer(call_time); - - gni_ep_handle_t ep_hdl; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - nnti_gni_work_request_t *gni_wr=NULL; - - uint64_t remaining=0; - - bool use_fma=true; - - struct { - uint32_t segment_index; // index of the current segment - uint64_t segment_offset; // offset into the current segment - uint64_t segment_remaining; // bytes remaining in the current segment - uint32_t last_segment; // index of the last segment in this transfer - } src_params, dst_params, src_copy, dst_copy; - - uint32_t sge_index=0; - - log_debug(nnti_ee_debug_level, "enter"); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - if (src_offset > src_buffer_hdl->payload_size) { - } - if (src_buffer_hdl->payload_size < src_length-src_offset) { - } - if (dest_offset > dest_buffer_hdl->payload_size) { - } - if (dest_buffer_hdl->payload_size < src_length-dest_offset) { - } - - gni_mem_hdl=(nnti_gni_memory_handle_t *)dest_buffer_hdl->transport_private; - assert(gni_mem_hdl); - - if (config.use_wr_pool) { - gni_wr=wr_pool_initiator_pop(); - } else { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - } - assert(gni_wr); - - gni_wr->reg_buf=dest_buffer_hdl; - gni_wr->last_op=GNI_OP_GET_INITIATOR; - gni_wr->peer_instance=src_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.gni.inst_id; - log_debug(nnti_event_debug_level, "gni_wr->peer_instance==%lu", gni_wr->peer_instance); - - gni_wr->nnti_wr=wr; - - wr->transport_id =dest_buffer_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)dest_buffer_hdl; - wr->ops =NNTI_BOP_LOCAL_WRITE; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)gni_wr; - - - nnti_gni_connection_t *conn=get_conn_peer(&src_buffer_hdl->buffer_owner); - assert(conn); - ep_hdl=conn->ep_hdl; - - nthread_lock(&gni_mem_hdl->wr_queue_lock); - gni_mem_hdl->wr_queue->push_back(gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - // the dest_offset could exclude some local segments from this operation. find the first segment. - dst_params.segment_offset =dest_offset; - dst_params.segment_remaining=0; - dst_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "dest_offset=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dest_buffer.segment[%llu].size==%llu", - dest_offset, dst_params.segment_offset, dst_params.segment_remaining, - (uint64_t)i, (uint64_t)dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size); - if (dst_params.segment_offset > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - dst_params.segment_offset -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - - log_debug(nnti_debug_level, "dst_params.segment_index=%lu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - continue; - } else { - dst_params.segment_index = i; - dst_params.segment_remaining = dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size - dst_params.segment_offset; - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=dst_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - remaining -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - } else { - dst_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dst_params.last_segment=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining, (uint64_t)dst_params.last_segment); - - // the src_offset could exclude some local segments from this operation. find the first segment. - src_params.segment_offset =src_offset; - src_params.segment_remaining=0; - src_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "src_offset=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_buffer.segment[%llu].size==%llu", - src_offset, src_params.segment_offset, src_params.segment_remaining, - (uint64_t)i, (uint64_t)src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size); - if (src_params.segment_offset > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - src_params.segment_offset -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - continue; - } else { - src_params.segment_index = i; - src_params.segment_remaining = src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size - src_params.segment_offset; - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=src_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size) { - remaining -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.gni.size; - } else { - src_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_params.last_segment=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining, (uint64_t)src_params.last_segment); - - /* START calculate the number of SGEs required */ - { - dst_copy=dst_params; - src_copy=src_params; - - NNTI_remote_addr_t *dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_copy.segment_index]; - NNTI_remote_addr_t *src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_copy.segment_index]; - - gni_wr->sge_count=0; - - while (src_copy.segment_index <= src_copy.last_segment) { - - gni_wr->sge_count++; - - if (src_copy.segment_remaining < dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment with extra space in the destination"); - // the remaining source segment fits in the remaining destination segment with extra space - dst_copy.segment_offset += src_copy.segment_remaining; - dst_copy.segment_remaining -= src_copy.segment_remaining; - - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else if (src_copy.segment_remaining == dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment EXACTLY"); - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else { - log_debug(nnti_debug_level, "the remaining source segment DOES NOT fit in the remaining destination segment"); - // the remaining source segment DOES NOT fit in the remaining destination segment - src_copy.segment_offset += dst_copy.segment_remaining; - src_copy.segment_remaining -= dst_copy.segment_remaining; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } - - log_debug(nnti_debug_level, "src_copy.segment_index=%llu, src_copy.segment_offset=%llu, src_copy.segment_remaining=%llu, src_copy.last_segment=%llu", - (uint64_t)src_copy.segment_index, src_copy.segment_offset, (uint64_t)src_copy.segment_remaining, (uint64_t)src_copy.last_segment); - log_debug(nnti_debug_level, "dst_copy.segment_index=%llu, dst_copy.segment_offset=%llu, dst_copy.segment_remaining=%llu, dst_copy.last_segment=%llu", - (uint64_t)dst_copy.segment_index, dst_copy.segment_offset, (uint64_t)dst_copy.segment_remaining, (uint64_t)dst_copy.last_segment); - } - } - /* END calculate the number of SGEs required */ - - NNTI_remote_addr_t *dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_params.segment_index]; - NNTI_remote_addr_t *src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_params.segment_index]; - - gni_wr->sge_list = (nnti_gni_sge_t *)malloc(gni_wr->sge_count*sizeof(nnti_gni_sge_t)); - - log_debug(nnti_debug_level, "this get requires %d SGEs", gni_wr->sge_count); - - sge_index=0; - - while (src_params.segment_index <= src_params.last_segment) { - - memset(&gni_wr->sge_list[sge_index].post_desc, 0, sizeof(gni_post_descriptor_t)); - - gni_wr->sge_list[sge_index].post_desc.src_cq_hndl=transport_global_data.ep_cq_hdl; - - gni_wr->sge_list[sge_index].post_desc.local_addr =dst_remote_addr->NNTI_remote_addr_t_u.gni.buf+dst_params.segment_offset; - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword1 =dst_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword1; - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword2 =dst_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword2; - - gni_wr->sge_list[sge_index].post_desc.remote_addr =src_remote_addr->NNTI_remote_addr_t_u.gni.buf+src_params.segment_offset; - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword1=src_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword1; - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword2=src_remote_addr->NNTI_remote_addr_t_u.gni.mem_hdl.qword2; - - if (src_params.segment_remaining <= dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - gni_wr->sge_list[sge_index].post_desc.length=src_params.segment_remaining; - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - gni_wr->sge_list[sge_index].post_desc.length=dst_params.segment_remaining; - } - - set_post_desc(&gni_wr->sge_list[sge_index].post_desc, GNI_OP_GET_INITIATOR, gni_wr->sge_list[sge_index].post_desc.length); - - gni_wr->sge_list[sge_index].state=NNTI_GNI_SGE_STATE_STARTED; - - gni_wr->sge_list[sge_index].gni_wr=gni_wr; - - gni_wr->wc=&gni_wr->rdma_wc; - gni_wr->wc->op =GNI_OP_GET_TARGET; - gni_wr->wc->byte_len =gni_wr->sge_list[sge_index].post_desc.length; - gni_wr->wc->src_index =src_params.segment_index; - gni_wr->wc->src_offset =src_params.segment_offset; - gni_wr->wc->dest_index =dst_params.segment_index; - gni_wr->wc->dest_offset=dst_params.segment_offset; - gni_wr->wc->inst_id =transport_global_data.instance; - - if (config.rdma_mode==RDMA_CROSSOVER) { - if (gni_wr->sge_list[sge_index].post_desc.length > config.fma_bte_crossover_size) { - use_fma=false; - } else { - use_fma=true; - } - } else if ((config.rdma_mode==RDMA_BTE) || (config.rdma_mode==RDMA_MIXED)) { - use_fma=false; - } else if (config.rdma_mode==RDMA_FMA) { - use_fma=true; - } - - insert_sge_sgehash(&gni_wr->sge_list[sge_index]); - - nthread_lock(&nnti_gni_lock); - GNI_EpSetEventData( - ep_hdl, - hash6432shift((uint64_t)&gni_wr->sge_list[sge_index]), - hash6432shift((uint64_t)src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - - if (use_fma) { - log_debug(nnti_event_debug_level, "calling PostFma(fma get ; ep_hdl(%llu) transport_global_data.ep_cq_hdl(%llu) local_mem_hdl(%llu, %llu) remote_mem_hdl(%llu, %llu))", - ep_hdl, transport_global_data.ep_cq_hdl, - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword2, - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword2); - trios_start_timer(call_time); - rc=GNI_PostFma(ep_hdl, &gni_wr->sge_list[sge_index].post_desc); - trios_stop_timer("PostFma get", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "failed to post FMA (rc=%d): %s", rc, strerror(errno)); - rc=NNTI_EIO; - } - } else { - log_debug(nnti_event_debug_level, "calling PostRdma(rdma get ; ep_hdl(%llu) transport_global_data.ep_cq_hdl(%llu) local_mem_hdl(%llu, %llu) remote_mem_hdl(%llu, %llu))", - ep_hdl, transport_global_data.ep_cq_hdl, - gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.local_mem_hndl.qword2, - gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword1, gni_wr->sge_list[sge_index].post_desc.remote_mem_hndl.qword2); - trios_start_timer(call_time); - rc=GNI_PostRdma(ep_hdl, &gni_wr->sge_list[sge_index].post_desc); - trios_stop_timer("PostRdma get", call_time); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "failed to post RDMA (rc=%d): %s", rc, strerror(errno)); - rc=NNTI_EIO; - } - } - nthread_unlock(&nnti_gni_lock); - - if (src_params.segment_remaining < dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - dst_params.segment_offset += src_params.segment_remaining; - dst_params.segment_remaining -= src_params.segment_remaining; - - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else if (src_params.segment_remaining == dst_params.segment_remaining) { - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.gni.size; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - src_params.segment_offset += dst_params.segment_remaining; - src_params.segment_remaining -= dst_params.segment_remaining; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.gni.size; - - } - - sge_index++; - } - - log_debug(nnti_ee_debug_level, "exit (wr=%p ; gni_wr=%p)", wr, gni_wr); - - return((NNTI_result_t)rc); -} - - -/** - * @brief Transfer data to a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to put. - * \param[in] src_length The number of bytes to put. - * \param[in] dest_buffer_list A list of buffers to put the data into. - * \param[in] dest_count The number of destination buffers. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_gni_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -/** - * @brief Transfer data from a peer. - * - * \param[in] src_buffer_list A list of buffers containing the data to get. - * \param[in] src_length The number of bytes to get. - * \param[in] src_count The number of source buffers. - * \param[in] dest_buffer_hdl A buffer to get the data into. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_gni_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_gni_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_gni_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value) -{ - nthread_lock(&transport_global_data.atomics_lock); - *value = transport_global_data.atomics[local_atomic]; - nthread_unlock(&transport_global_data.atomics_lock); - - return NNTI_OK; -} - - -NNTI_result_t NNTI_gni_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr) -{ - int rc=0; - - nnti_gni_work_request_t *gni_wr=NULL; - - log_level debug_level=nnti_debug_level; - - log_debug(nnti_ee_debug_level, "enter"); - - if (config.use_wr_pool) { - gni_wr=wr_pool_initiator_pop(); - } else { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - } - assert(gni_wr); - - gni_wr->reg_buf = NULL; - - nnti_gni_connection_t *conn=get_conn_peer(peer_hdl); - assert(conn); - - gni_wr->last_op=GNI_OP_FETCH_ADD; - gni_wr->peer_instance=peer_hdl->peer.NNTI_remote_process_t_u.gni.inst_id; - log_debug(nnti_event_debug_level, "gni_wr->peer_instance==%lu", gni_wr->peer_instance); - - gni_wr->sge_count=1; - gni_wr->sge_list =&gni_wr->sge; - - memset(&gni_wr->sge_list[0].post_desc, 0, sizeof(gni_post_descriptor_t)); - - gni_wr->sge_list[0].post_desc.src_cq_hndl=transport_global_data.ep_cq_hdl; - - gni_wr->sge_list[0].post_desc.type =GNI_POST_AMO; - gni_wr->sge_list[0].post_desc.cq_mode=GNI_CQMODE_GLOBAL_EVENT; - - set_dlvr_mode(&gni_wr->sge_list[0].post_desc); - set_rdma_mode(&gni_wr->sge_list[0].post_desc); - - gni_wr->sge_list[0].post_desc.local_addr =(uint64_t)transport_global_data.atomics+(result_atomic*sizeof(int64_t)); - gni_wr->sge_list[0].post_desc.local_mem_hndl =transport_global_data.atomics_mem_hdl; - gni_wr->sge_list[0].post_desc.remote_addr =conn->atomics_addr+(target_atomic*sizeof(int64_t)); - gni_wr->sge_list[0].post_desc.remote_mem_hndl=conn->atomics_mem_hdl; - gni_wr->sge_list[0].post_desc.length =sizeof(int64_t); - gni_wr->sge_list[0].post_desc.amo_cmd =GNI_FMA_ATOMIC_FADD; - gni_wr->sge_list[0].post_desc.first_operand =operand; - gni_wr->sge_list[0].post_desc.second_operand =0; - - gni_wr->sge_list[0].state=NNTI_GNI_SGE_STATE_STARTED; - gni_wr->sge_list[0].gni_wr=gni_wr; - - gni_wr->nnti_wr = wr; - - wr->transport_id =trans_hdl->id; - wr->reg_buf =(NNTI_buffer_t*)NULL; - wr->ops =NNTI_BOP_ATOMICS; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)gni_wr; - - insert_sge_sgehash(&gni_wr->sge_list[0]); - - log_debug(debug_level, "calling PostFma(atomics fop - ep_hdl(%llu), local_addr=%llu, remote_addr=%llu)", - conn->ep_hdl, gni_wr->sge_list[0].post_desc.local_addr, (uint64_t)gni_wr->sge_list[0].post_desc.remote_addr); - - nthread_lock(&nnti_gni_lock); - GNI_EpSetEventData( - conn->ep_hdl, - hash6432shift((uint64_t)&gni_wr->sge_list[0]), - 0); - rc=GNI_PostFma(conn->ep_hdl, &gni_wr->sge_list[0].post_desc); - if (rc!=GNI_RC_SUCCESS) log_error(debug_level, "PostFma(atomics fop) failed: %d", rc); - nthread_unlock(&nnti_gni_lock); - - return(NNTI_OK); -} - - -NNTI_result_t NNTI_gni_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr) -{ - int rc=0; - - nnti_gni_work_request_t *gni_wr=NULL; - - log_level debug_level=nnti_debug_level; - - log_debug(nnti_ee_debug_level, "enter"); - - if (config.use_wr_pool) { - gni_wr=wr_pool_initiator_pop(); - } else { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - } - assert(gni_wr); - - gni_wr->reg_buf = NULL; - - nnti_gni_connection_t *conn=get_conn_peer(peer_hdl); - assert(conn); - - gni_wr->last_op=GNI_OP_COMPARE_SWAP; - gni_wr->peer_instance=peer_hdl->peer.NNTI_remote_process_t_u.gni.inst_id; - log_debug(nnti_event_debug_level, "gni_wr->peer_instance==%lu", gni_wr->peer_instance); - - gni_wr->sge_count=1; - gni_wr->sge_list =&gni_wr->sge; - - memset(&gni_wr->sge_list[0].post_desc, 0, sizeof(gni_post_descriptor_t)); - - gni_wr->sge_list[0].post_desc.src_cq_hndl=transport_global_data.ep_cq_hdl; - - gni_wr->sge_list[0].post_desc.type =GNI_POST_AMO; - gni_wr->sge_list[0].post_desc.cq_mode=GNI_CQMODE_GLOBAL_EVENT; - - set_dlvr_mode(&gni_wr->sge_list[0].post_desc); - set_rdma_mode(&gni_wr->sge_list[0].post_desc); - - gni_wr->sge_list[0].post_desc.local_addr =(uint64_t)transport_global_data.atomics+(result_atomic*sizeof(int64_t)); - gni_wr->sge_list[0].post_desc.local_mem_hndl =transport_global_data.atomics_mem_hdl; - gni_wr->sge_list[0].post_desc.remote_addr =conn->atomics_addr+(target_atomic*sizeof(int64_t)); - gni_wr->sge_list[0].post_desc.remote_mem_hndl=conn->atomics_mem_hdl; - gni_wr->sge_list[0].post_desc.length =sizeof(int64_t); - gni_wr->sge_list[0].post_desc.amo_cmd =GNI_FMA_ATOMIC_CSWAP; - gni_wr->sge_list[0].post_desc.first_operand =compare_operand; - gni_wr->sge_list[0].post_desc.second_operand =swap_operand; - - gni_wr->sge_list[0].state=NNTI_GNI_SGE_STATE_STARTED; - gni_wr->sge_list[0].gni_wr=gni_wr; - - gni_wr->nnti_wr = wr; - - wr->transport_id =trans_hdl->id; - wr->reg_buf =(NNTI_buffer_t*)NULL; - wr->ops =NNTI_BOP_ATOMICS; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)gni_wr; - - insert_sge_sgehash(&gni_wr->sge_list[0]); - - log_debug(debug_level, "calling PostFma(atomics cswap - ep_hdl(%llu), local_addr=%llu, remote_addr=%llu)", - conn->ep_hdl, gni_wr->sge_list[0].post_desc.local_addr, (uint64_t)gni_wr->sge_list[0].post_desc.remote_addr); - - nthread_lock(&nnti_gni_lock); - GNI_EpSetEventData( - conn->ep_hdl, - hash6432shift((uint64_t)&gni_wr->sge_list[0]), - 0); - rc=GNI_PostFma(conn->ep_hdl, &gni_wr->sge_list[0].post_desc); - if (rc!=GNI_RC_SUCCESS) log_error(debug_level, "PostFma(atomics cswap) failed: %d", rc); - nthread_unlock(&nnti_gni_lock); - - return(NNTI_OK); -} - - -/** - * @brief Create a receive work request that can be used to wait for buffer - * operations to complete. - * - */ -NNTI_result_t NNTI_gni_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr) -{ - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - nnti_gni_work_request_t *gni_wr=NULL; - - bool found=false; - - gni_mem_hdl=GNI_MEM_HDL(reg_buf); - assert(gni_mem_hdl); - - log_debug(nnti_debug_level, "enter (reg_buf=%p ; wr=%p)", reg_buf, wr); - - if (reg_buf->ops == NNTI_BOP_RECV_QUEUE) { - nnti_gni_request_queue_handle_t *q_hdl=&transport_global_data.req_queue; - assert(q_hdl); - - // head and tail are NOT adjacent. search between them for a work request in the NNTI_RESET state. - if (q_hdl->tail <= q_hdl->head) { - // head has not wrapped around yet. simple forward search. - for (int64_t index=q_hdl->tail; index<=q_hdl->head; index++) { - gni_wr=gni_mem_hdl->wr_queue->at(index); - - log_debug(nnti_debug_level, "wr_queue[%lld]==%p", index, gni_wr); - - log_debug(nnti_debug_level, "gni_wr(%p) gni_wr->state=%d", gni_wr, gni_wr->state); - if ((gni_wr->state==NNTI_GNI_WR_STATE_POSTED) || (gni_wr->state==NNTI_GNI_WR_STATE_RDMA_COMPLETE)) { - GNI_ATTACH_WR(wr,gni_wr); - - if (index == q_hdl->head) { - q_hdl->head = (q_hdl->head+1) % q_hdl->req_count; - log_debug(nnti_debug_level, "new head==%lld", q_hdl->head); - } - - found=true; - - break; - } - } - } else { - // head has wrapped around. search from tail to end, then front to head. - for (int64_t index=q_hdl->tail; indexreq_count; index++) { - gni_wr=gni_mem_hdl->wr_queue->at(index); - - log_debug(nnti_debug_level, "wr_queue[%lld]==%p", index, gni_wr); - - log_debug(nnti_debug_level, "gni_wr(%p) gni_wr->state=%d", gni_wr, gni_wr->state); - if ((gni_wr->state==NNTI_GNI_WR_STATE_POSTED) || (gni_wr->state==NNTI_GNI_WR_STATE_RDMA_COMPLETE)) { - GNI_ATTACH_WR(wr,gni_wr); - - found=true; - - break; - } - } - if (!found) { - // didn't find anything at the end of the queue. look at the front. - for (int64_t index=0; index<=q_hdl->head; index++) { - gni_wr=gni_mem_hdl->wr_queue->at(index); - - log_debug(nnti_debug_level, "wr_queue[%lld]==%p", index, gni_wr); - - log_debug(nnti_debug_level, "gni_wr(%p) gni_wr->state=%d", gni_wr, gni_wr->state); - if ((gni_wr->state==NNTI_GNI_WR_STATE_POSTED) || (gni_wr->state==NNTI_GNI_WR_STATE_RDMA_COMPLETE)) { - GNI_ATTACH_WR(wr,gni_wr); - - if (index == q_hdl->head) { - q_hdl->head = (q_hdl->head+1) % q_hdl->req_count; - log_debug(nnti_debug_level, "new head==%lld", q_hdl->head); - } - - found=true; - - break; - } - } - } - } - log_debug(nnti_debug_level, "q->head=%lld ; q->tail==%lld", q_hdl->head, q_hdl->tail); - } else if (reg_buf->ops & NNTI_BOP_WITH_EVENTS) { - gni_wr=gni_mem_hdl->wr_queue->front(); - GNI_ATTACH_WR(wr,gni_wr); - } else { - - } - - wr->transport_id =reg_buf->transport_id; - wr->reg_buf =reg_buf; - wr->ops =reg_buf->ops; - wr->result =NNTI_OK; - - log_debug(nnti_debug_level, "assigned nnti_gni_work_request_t(%p) to NNTI_work_request(%p)", gni_wr, wr); - - log_debug(nnti_debug_level, "exit (reg_buf=%p ; wr=%p)", reg_buf, wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from a previous receive - * and prepares it for reuse. - * - */ -NNTI_result_t NNTI_gni_clear_work_request ( - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - wr->result =NNTI_OK; - wr->transport_private=0; - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from reg_buf. - * - */ -NNTI_result_t NNTI_gni_destroy_work_request ( - NNTI_work_request_t *wr) -{ - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - nnti_gni_work_request_t *gni_wr =NULL; - nnti_gni_work_request_t *tail_wr =NULL; - - wr_queue_iter_t q_victim; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - gni_mem_hdl=GNI_MEM_HDL(wr->reg_buf); - assert(gni_mem_hdl); - gni_wr=GNI_WORK_REQUEST(wr); - - if (!gni_wr) { - return(NNTI_OK); - } - - if ((gni_wr->last_op == GNI_OP_SEND_REQUEST) || - (gni_wr->last_op == GNI_OP_SEND_BUFFER) || - (gni_wr->last_op == GNI_OP_GET_INITIATOR) || - (gni_wr->last_op == GNI_OP_PUT_INITIATOR)) { - // work requrests from initiator buffers are cleaned up in NNTI_gni_wait*(). nothing to do here. - return(NNTI_OK); - } - - if (wr->ops == NNTI_BOP_RECV_QUEUE) { - nnti_gni_request_queue_handle_t *q_hdl=&transport_global_data.req_queue; - assert(q_hdl); - - if (gni_wr->state == NNTI_GNI_WR_STATE_ATTACHED) { - gni_wr->state=NNTI_GNI_WR_STATE_POSTED; - } else { - tail_wr=gni_mem_hdl->wr_queue->at(q_hdl->tail); - assert(tail_wr); - - if (gni_wr == tail_wr) { - if (q_hdl->tail <= q_hdl->head) { - // head has not wrapped around yet. move tail forward to the next work request NOT in the NNTI_COMPLETE state. - for (int64_t index=q_hdl->tail; indexhead; index++) { - gni_wr=gni_mem_hdl->wr_queue->at(index); - if (gni_wr->state == NNTI_GNI_WR_STATE_WAIT_COMPLETE) { - gni_wr->state=NNTI_GNI_WR_STATE_POSTED; - - q_hdl->tail = (q_hdl->tail+1) % q_hdl->req_count; - } else { - break; - } - } - } else { - // head has wrapped around. search from tail to end, then front to head. - for (int64_t index=q_hdl->tail; indexreq_count; index++) { - gni_wr=gni_mem_hdl->wr_queue->at(index); - if (gni_wr->state == NNTI_GNI_WR_STATE_WAIT_COMPLETE) { - gni_wr->state=NNTI_GNI_WR_STATE_POSTED; - - q_hdl->tail = (q_hdl->tail+1) % q_hdl->req_count; - } else { - break; - } - } - if (q_hdl->tail == 0) { - // tail wrapped around in the previous loop. keep going at the front. - for (int64_t index=0; indexhead; index++) { - gni_wr=gni_mem_hdl->wr_queue->at(index); - if (gni_wr->state == NNTI_GNI_WR_STATE_WAIT_COMPLETE) { - gni_wr->state=NNTI_GNI_WR_STATE_POSTED; - - q_hdl->tail = (q_hdl->tail+1) % q_hdl->req_count; - } else { - break; - } - } - } - } - } - } - if ((nthread_counter_read(&q_hdl->req_index) >= q_hdl->req_count) && - (q_hdl->head == 0) && - (q_hdl->tail == 0)) { - - nthread_counter_set(&q_hdl->req_index, 0); - log_debug(nnti_event_debug_level, "resetting req_index(%lld)", nthread_counter_read(&q_hdl->req_index)); - } - - log_debug(nnti_debug_level, "q->head=%lld ; q->tail==%lld", q_hdl->head, q_hdl->tail); - } else if (wr->ops & NNTI_BOP_WITH_EVENTS) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - gni_wr->state=NNTI_GNI_WR_STATE_POSTED; - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), gni_wr); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", gni_wr); - gni_mem_hdl->wr_queue->erase(q_victim); - } - repost_recv_work_request(wr->reg_buf, gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - } - - GNI_DETACH_WR(wr, gni_wr); - - wr->transport_id=NNTI_TRANSPORT_NULL; - wr->reg_buf =NULL; - wr->ops =(NNTI_buf_ops_t)0; - wr->result =NNTI_OK; - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Attempts to cancel an NNTI opertion. - * - */ -NNTI_result_t NNTI_gni_cancel ( - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -/** - * @brief Attempts to cancel a list of NNTI opertions. - * - */ -NNTI_result_t NNTI_gni_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -/** - * @brief Interrupts NNTI_wait*() - * - */ -NNTI_result_t NNTI_gni_interrupt ( - const NNTI_transport_t *trans_hdl) -{ - NNTI_result_t rc=NNTI_OK; - int gni_rc=GNI_RC_SUCCESS; - - uint32_t dummy=0xAAAAAAAA; - uint32_t *ptr32=NULL; - gni_post_descriptor_t post_desc; - gni_post_descriptor_t *post_desc_ptr; - gni_cq_entry_t ev_data; - - log_debug(nnti_debug_level, "enter"); - - memset(&post_desc, 0, sizeof(gni_post_descriptor_t)); - post_desc.type =GNI_POST_CQWRITE; - post_desc.cq_mode =GNI_CQMODE_GLOBAL_EVENT|GNI_CQMODE_REMOTE_EVENT; - post_desc.dlvr_mode =GNI_DLVMODE_IN_ORDER; - post_desc.remote_mem_hndl=transport_global_data.interrupt_mem_hdl; - - ptr32=(uint32_t*)&post_desc.cqwrite_value; - *ptr32=dummy; - - log_debug(nnti_debug_level, "calling PostCqWrite(cqwrite_value=%X)\n", *(uint32_t*)&post_desc.cqwrite_value); - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_PostCqWrite(transport_global_data.interrupt_ep_hdl, &post_desc); - nthread_unlock(&nnti_gni_lock); - if (gni_rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "PostCqWrite(post_desc) failed: %d\n", rc); - rc=NNTI_EIO; - goto cleanup; - } - - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_CqWaitEvent(transport_global_data.interrupt_ep_cq_hdl, -1, &ev_data); - DEQUEUE_POST_DESCRIPTOR(transport_global_data.interrupt_ep_cq_hdl, &ev_data, &post_desc_ptr); - nthread_unlock(&nnti_gni_lock); - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return rc; -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t NNTI_gni_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - nnti_gni_work_request_t *gni_wr=NULL; - - wr_queue_iter_t q_victim; - - gni_cq_entry_t ev_data; - - NNTI_result_t rc=NNTI_OK; - long elapsed_time = 0; - long timeout_per_call = timeout; - - long entry_time=trios_get_time_ms(); - - log_debug(nnti_ee_debug_level, "enter (wr=%p)", wr); - - assert(wr); - assert(status); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "wr->reg_buf", - "start of NNTI_gni_wait", wr->reg_buf); - } - - gni_wr=GNI_WORK_REQUEST(wr); - if (gni_wr == NULL) { - // this work request is "inactive". - nnti_rc=NNTI_EINVAL; - goto cleanup; - } - - check_listen_socket_for_new_connections(); - - if ((config.max_timeout_ms > 0) && (config.max_timeout_ms < timeout)) { - timeout_per_call=config.max_timeout_ms; - } - - if (wr->result == NNTI_EDROPPED) { - log_debug(nnti_debug_level, "send was dropped (wr=%p ; gni_wr=%p)", wr, GNI_WORK_REQUEST(wr)); - nnti_rc = NNTI_EDROPPED; - } else if (is_wr_canceling(gni_wr)) { - log_debug(nnti_debug_level, "wr canceling (wr=%p ; gni_wr=%p)", wr, GNI_WORK_REQUEST(wr)); - cancel_wr(gni_wr); - } else if (is_wr_complete(gni_wr) == TRUE) { - log_debug(nnti_debug_level, "wr already complete (wr=%p ; gni_wr=%p)", wr, GNI_WORK_REQUEST(wr)); - nnti_rc = NNTI_OK; - } else { - log_debug(nnti_debug_level, "wr NOT complete (wr=%p ; gni_wr=%p)", wr, GNI_WORK_REQUEST(wr)); - - while (1) { - rc=progress(timeout_per_call, &wr, 1); - - elapsed_time = (trios_get_time_ms() - entry_time); - - /* case 1: success */ - if (rc==NNTI_OK) { -// logger_set_default_level(old_log_level); - log_debug(nnti_debug_level, "progress() successful..."); - nnti_rc = rc; - } - /* case 2: timed out */ - else if (rc==NNTI_ETIMEDOUT) { - log_debug(nnti_debug_level, "progress() timed out..."); -// logger_set_default_level(LOG_OFF); - } - /* case 3: interrupted */ - else if (rc==NNTI_EINTR) { - log_debug(nnti_debug_level, "progress() interrupted..."); -// logger_set_default_level(LOG_OFF); - nnti_rc = rc; - break; - } - /* case 4: failure */ - else { -// logger_set_default_level(old_log_level); - log_debug(nnti_debug_level, "progress() failed: %s", strerror(errno)); - nnti_rc = rc; - break; - } - - resend_all(); - execute_mbox_backlog(); - - if (is_wr_complete(gni_wr) == TRUE) { - log_debug(nnti_debug_level, "wr completed (wr=%p ; gni_wr=%p)", wr, GNI_WORK_REQUEST(wr)); - nnti_rc = NNTI_OK; - break; - } - - /* if the caller asked for a legitimate timeout, we need to exit */ - if ((timeout >= 0) && (elapsed_time >= timeout)) { - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - if (trios_exit_now()) { - log_debug(nnti_debug_level, "caught abort signal"); - nnti_rc = NNTI_ECANCELED; - break; - } - } - } - - create_status(wr, gni_wr, nnti_rc, &ev_data, status); - -// print_wc(&wc); -// if (nnti_rc == NNTI_OK) { -// print_raw_buf((char *)reg_buf->payload+gni_wr->wc.byte_offset, gni_mem_hdl->last_wc.byte_len); -// } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_wait", status); - } - - if (nnti_rc == NNTI_OK) { - - gni_wr->state=NNTI_GNI_WR_STATE_WAIT_COMPLETE; - - if (gni_wr->nnti_wr->ops == NNTI_BOP_ATOMICS) { - del_sge_sgehash(&gni_wr->sge_list[0]); - - if (config.use_wr_pool) { - wr_pool_initiator_push(gni_wr); - } else { - log_debug(nnti_debug_level, "free(gni_wr) (wr=%p, gni_wr=%p)", wr, gni_wr); - free(gni_wr); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr->transport_private=(uint64_t)NULL; - - } else { - gni_mem_hdl=GNI_MEM_HDL(wr->reg_buf); - assert(gni_mem_hdl); - - if ((gni_wr->last_op == GNI_OP_NEW_REQUEST) || - (gni_wr->last_op == GNI_OP_RECEIVE)) { - // defer cleanup to NNTI_gni_destroy_work_request() - } - else if ((gni_wr->last_op == GNI_OP_GET_TARGET) || - (gni_wr->last_op == GNI_OP_PUT_TARGET)) { - if (config.use_rdma_target_ack) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), gni_wr); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", gni_wr); - gni_mem_hdl->wr_queue->erase(q_victim); - } - repost_recv_work_request(wr->reg_buf, gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - } - } - else if ((gni_wr->last_op == GNI_OP_SEND_REQUEST) || - (gni_wr->last_op == GNI_OP_SEND_BUFFER) || - (gni_wr->last_op == GNI_OP_GET_INITIATOR) || - (gni_wr->last_op == GNI_OP_PUT_INITIATOR)) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), gni_wr); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", gni_wr); - gni_mem_hdl->wr_queue->erase(q_victim); - } - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - for (uint32_t j=0;jsge_count;j++) { - del_sge_sgehash(&gni_wr->sge_list[j]); - } - if (gni_wr->sge_count > 1) { - free(gni_wr->sge_list); - } - if (config.use_wr_pool) { - wr_pool_initiator_push(gni_wr); - } else { - log_debug(nnti_debug_level, "free(gni_wr) (wr=%p, gni_wr=%p)", wr, gni_wr); - free(gni_wr); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr->transport_private=(uint64_t)NULL; - } - } - } - -cleanup: - log_debug(nnti_ee_debug_level, "exit (wr=%p)", wr); - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on any buffer in buf_list to complete. - * - * Wait for remote_op on any buffer in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the request queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_gni_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - wr_queue_iter_t q_victim; - - nnti_gni_work_request_t *gni_wr=NULL; - - gni_cq_entry_t ev_data; - - NNTI_result_t rc=NNTI_OK; - long elapsed_time = 0; - long timeout_per_call=timeout; - - long entry_time=trios_get_time_ms(); - - log_debug(nnti_ee_debug_level, "enter"); - - check_listen_socket_for_new_connections(); - - assert(wr_list); - assert(wr_count > 0); - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_gni_wait(wr_list[0], timeout, status); - *which=0; - goto cleanup; - } - - if ((config.max_timeout_ms > 0) && (config.max_timeout_ms < timeout)) { - timeout_per_call=config.max_timeout_ms; - } - - if (wr_count > 1) { - for (uint32_t i=0;i= 0) && (elapsed_time >= timeout)) { - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - if (trios_exit_now()) { - log_debug(nnti_debug_level, "caught abort signal"); - nnti_rc = NNTI_ECANCELED; - break; - } - } - } - - create_status(wr_list[*which], GNI_WORK_REQUEST(wr_list[*which]), nnti_rc, &ev_data, status); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_waitany", status); - } - - if (is_wr_complete(GNI_WORK_REQUEST(wr_list[*which]))) { - - GNI_WORK_REQUEST(wr_list[*which])->state=NNTI_GNI_WR_STATE_WAIT_COMPLETE; - - gni_mem_hdl=GNI_MEM_HDL(wr_list[*which]->reg_buf); - assert(gni_mem_hdl); - - if ((gni_wr->last_op == GNI_OP_NEW_REQUEST) || - (gni_wr->last_op == GNI_OP_RECEIVE)) { - // defer cleanup to NNTI_gni_destroy_work_request() - } - else if ((gni_wr->last_op == GNI_OP_GET_TARGET) || - (gni_wr->last_op == GNI_OP_PUT_TARGET)) { - if (config.use_rdma_target_ack) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), GNI_WORK_REQUEST(wr_list[*which])); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", GNI_WORK_REQUEST(wr_list[*which])); - gni_mem_hdl->wr_queue->erase(q_victim); - } - repost_recv_work_request(wr_list[*which]->reg_buf, GNI_WORK_REQUEST(wr_list[*which])); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - } - } - else if ((gni_wr->last_op == GNI_OP_SEND_REQUEST) || - (gni_wr->last_op == GNI_OP_SEND_BUFFER) || - (gni_wr->last_op == GNI_OP_GET_INITIATOR) || - (gni_wr->last_op == GNI_OP_PUT_INITIATOR)) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), GNI_WORK_REQUEST(wr_list[*which])); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", GNI_WORK_REQUEST(wr_list[*which])); - gni_mem_hdl->wr_queue->erase(q_victim); - } - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - for (uint32_t j=0;jsge_count;j++) { - del_sge_sgehash(&GNI_WORK_REQUEST(wr_list[*which])->sge_list[j]); - } - if (GNI_WORK_REQUEST(wr_list[*which])->sge_count > 1) { - free(GNI_WORK_REQUEST(wr_list[*which])->sge_list); - } - if (config.use_wr_pool) { - wr_pool_initiator_push(GNI_WORK_REQUEST(wr_list[*which])); - } else { - free(GNI_WORK_REQUEST(wr_list[*which])); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr_list[*which]->transport_private=(uint64_t)NULL; - } - } - -cleanup: - log_debug(nnti_ee_debug_level, "exit"); - - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on all buffers in buf_list to complete. - * - * Wait for remote_op on all buffers in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the receive queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_gni_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - wr_queue_iter_t q_victim; - - nnti_gni_work_request_t *gni_wr=NULL; - - NNTI_result_t rc=NNTI_OK; - long elapsed_time = 0; - long timeout_per_call=timeout; - - long entry_time=trios_get_time_ms(); - - log_debug(nnti_ee_debug_level, "enter"); - - check_listen_socket_for_new_connections(); - - assert(wr_list); - assert(wr_count > 0); - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_gni_wait(wr_list[0], timeout, status[0]); - goto cleanup; - } - - if ((config.max_timeout_ms > 0) && (config.max_timeout_ms < timeout)) { - timeout_per_call=config.max_timeout_ms; - } - - if (wr_count > 1) { - for (uint32_t i=0;i= 0) && (elapsed_time >= timeout)) { - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - if (trios_exit_now()) { - log_debug(nnti_debug_level, "caught abort signal"); - nnti_rc = NNTI_ECANCELED; - break; - } - } - } - - for (uint32_t i=0;istate=NNTI_GNI_WR_STATE_WAIT_COMPLETE; - - gni_mem_hdl=GNI_MEM_HDL(wr_list[i]->reg_buf); - assert(gni_mem_hdl); - - if ((gni_wr->last_op == GNI_OP_NEW_REQUEST) || - (gni_wr->last_op == GNI_OP_RECEIVE)) { - // defer cleanup to NNTI_gni_destroy_work_request() - } - else if ((gni_wr->last_op == GNI_OP_GET_TARGET) || - (gni_wr->last_op == GNI_OP_PUT_TARGET)) { - if (config.use_rdma_target_ack) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), GNI_WORK_REQUEST(wr_list[i])); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", GNI_WORK_REQUEST(wr_list[i])); - gni_mem_hdl->wr_queue->erase(q_victim); - } - repost_recv_work_request(wr_list[i]->reg_buf, GNI_WORK_REQUEST(wr_list[i])); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - } - } - else if ((gni_wr->last_op == GNI_OP_SEND_REQUEST) || - (gni_wr->last_op == GNI_OP_SEND_BUFFER) || - (gni_wr->last_op == GNI_OP_GET_INITIATOR) || - (gni_wr->last_op == GNI_OP_PUT_INITIATOR)) { - nthread_lock(&gni_mem_hdl->wr_queue_lock); - q_victim=find(gni_mem_hdl->wr_queue->begin(), gni_mem_hdl->wr_queue->end(), GNI_WORK_REQUEST(wr_list[i])); - if (q_victim != gni_mem_hdl->wr_queue->end()) { - log_debug(nnti_debug_level, "erasing gni_wr=%p from the wr_queue", GNI_WORK_REQUEST(wr_list[i])); - gni_mem_hdl->wr_queue->erase(q_victim); - } - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - for (uint32_t j=0;jsge_count;j++) { - del_sge_sgehash(&GNI_WORK_REQUEST(wr_list[i])->sge_list[j]); - } - if (GNI_WORK_REQUEST(wr_list[i])->sge_count > 1) { - free(GNI_WORK_REQUEST(wr_list[i])->sge_list); - } - if (config.use_wr_pool) { - wr_pool_initiator_push(GNI_WORK_REQUEST(wr_list[i])); - } else { - free(GNI_WORK_REQUEST(wr_list[i])); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr_list[i]->transport_private=(uint64_t)NULL; - } - } - } - - -cleanup: - log_debug(nnti_ee_debug_level, "exit"); - - return(nnti_rc); -} - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t NNTI_gni_fini ( - const NNTI_transport_t *trans_hdl) -{ - int rc=GNI_RC_SUCCESS; /* return code */ - - log_debug(nnti_ee_debug_level, "enter"); - - close_all_conn(); - - if (config.use_wr_pool) { - rc=wr_pool_fini(); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "wr_pool_fini() failed: %d", rc); - rc=NNTI_EINVAL; - } - } - - close(transport_global_data.listen_sock); - transport_global_data.listen_name[0]='\0'; - transport_global_data.listen_addr=0; - transport_global_data.listen_port=0; - - rc=GNI_CdmDestroy(transport_global_data.cdm_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "CdmCreate() failed: %d", rc); - rc=NNTI_EINVAL; - } - - nthread_lock_fini(&nnti_gni_lock); - nthread_lock_fini(&nnti_resend_lock); - nthread_lock_fini(&nnti_mbox_backlog_lock); - nthread_lock_fini(&nnti_mem_lock); - - nthread_lock_fini(&nnti_progress_lock); - nthread_cond_fini(&nnti_progress_cond); - - nthread_lock_fini(&nnti_conn_peer_lock); - nthread_lock_fini(&nnti_conn_instance_lock); - nthread_lock_fini(&nnti_buf_bufhash_lock); - nthread_lock_fini(&nnti_sge_sgehash_lock); - nthread_lock_fini(&nnti_wr_pool_lock); - - nthread_lock_fini(&transport_global_data.atomics_lock); - - gni_initialized = false; - - log_debug(nnti_ee_debug_level, "exit"); - - return(NNTI_OK); -} - -static NNTI_result_t setup_atomics(void) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - int gni_rc=GNI_RC_SUCCESS; /* return code */ - - trios_declare_timer(call_time); - - uint32_t atomics_bytes; - - log_debug(nnti_debug_level, "enter"); - - atomics_bytes=config.min_atomics_vars * sizeof(int64_t); - trios_start_timer(call_time); - transport_global_data.atomics=(int64_t*)aligned_malloc(atomics_bytes); - if (transport_global_data.atomics == NULL) { - return(NNTI_ENOMEM); - } - memset(transport_global_data.atomics, 0, atomics_bytes); - trios_stop_timer("malloc and memset", call_time); - - trios_start_timer(call_time); - gni_rc=GNI_MemRegister (transport_global_data.nic_hdl, - (uint64_t)transport_global_data.atomics, - atomics_bytes, - NULL, - GNI_MEM_READWRITE, - (uint32_t)-1, - &transport_global_data.atomics_mem_hdl); - if (gni_rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "MemRegister(atomics_mem_hdl) failed: gni_rc=%d, %s", gni_rc, strerror(errno)); - rc=NNTI_EIO; - } - trios_stop_timer("atomics register", call_time); - - log_debug(nnti_debug_level, "exit (atomics==%p)...", transport_global_data.atomics); - - return(rc); -} - -static void *aligned_malloc( - size_t size) -{ - int rc = 0; - void * ptr = NULL; - -#ifdef NO_MEMALIGN - ptr = malloc( size ); -#else - rc = posix_memalign( &ptr , 64 , size ); -#endif - - if (rc!=0) { - log_error(nnti_debug_level, "posix_memalign() failed: rc=%d (%s)", rc, strerror(errno)); - ptr=NULL; - } - - return ptr; -} - -static gni_mem_handle_t register_memory_segment(NNTI_buf_ops_t ops, void *buf, uint64_t len, uint64_t extra) -{ - NNTI_result_t nnti_rc=NNTI_OK; - int gni_rc=GNI_RC_SUCCESS; /* return code */ - - uint32_t flags=GNI_MEM_READWRITE; - - trios_declare_timer(call_time); - - gni_mem_handle_t mem_hdl; - gni_cq_handle_t mem_cq_hdl; - - mem_cq_hdl=NULL; - - log_debug(nnti_debug_level, "enter ops(%d) buffer(%p) len(%llu) extra(%llu)", ops, buf, len, extra); - - if (config.pi_ordering==PI_ORDERING_STRICT) { - log_debug(nnti_debug_level, "using STRICT ordering"); - flags = GNI_MEM_READWRITE|GNI_MEM_STRICT_PI_ORDERING; - } else if (config.pi_ordering==PI_ORDERING_RELAXED) { - log_debug(nnti_debug_level, "using RELAXED ordering"); - flags = GNI_MEM_READWRITE|GNI_MEM_RELAXED_PI_ORDERING; - } else if (config.pi_ordering==PI_ORDERING_DEFAULT) { - log_debug(nnti_debug_level, "using DEFAULT ordering"); - flags = GNI_MEM_READWRITE; - } - - if (need_mem_cq(ops) == 1) { - mem_cq_hdl=transport_global_data.mem_cq_hdl; - } - - trios_start_timer(call_time); - gni_rc=GNI_MemRegister (transport_global_data.nic_hdl, - (uint64_t)buf, - len+extra, - mem_cq_hdl, - flags, - (uint32_t)-1, - &mem_hdl); - if (gni_rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "MemRegister(mem_hdl) failed: gni_rc=%d, %s", gni_rc, strerror(errno)); - goto cleanup; - } - trios_stop_timer("buf register", call_time); - - log_debug(nnti_debug_level, "register mem_cq_hdl =%llu", (uint64_t)mem_cq_hdl); - log_debug(nnti_debug_level, "register hdl->mem_hdl=(%llu,%llu)", (uint64_t)mem_hdl.qword1, (uint64_t)mem_hdl.qword2); - -cleanup: - switch(gni_rc) { - case GNI_RC_SUCCESS: - nnti_rc=NNTI_OK; - break; - default: - nnti_rc=NNTI_EIO; - break; - } - - log_debug(nnti_debug_level, "exit ops(%d) buffer(%p) gni_rc(%d) nnti_rc(%d)", ops, buf, gni_rc, nnti_rc); - - return (mem_hdl); -} - -static NNTI_result_t register_memory( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t extra, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - NNTI_buffer_t *old_buf=NULL; - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - assert(trans_hdl); - assert(buffer); - assert(element_size>0); - assert(extra>=0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - log_debug(nnti_ee_debug_level, "enter (reg_buf=%p)", reg_buf); - -nthread_lock(&nnti_mem_lock); - - old_buf=get_buf_bufhash(hash6432shift((uint64_t)buffer)); - if (old_buf==NULL) { - gni_mem_hdl=(nnti_gni_memory_handle_t*)calloc(1, sizeof(nnti_gni_memory_handle_t)); - assert(gni_mem_hdl); - gni_mem_hdl->wr_queue =new wr_queue_t; - gni_mem_hdl->ref_count=1; - nthread_lock_init(&gni_mem_hdl->wr_queue_lock); - } else { - gni_mem_hdl=(nnti_gni_memory_handle_t*)old_buf->transport_private; - gni_mem_hdl->ref_count++; - } - - log_debug(nnti_ee_debug_level, "gni_mem_hdl->ref_count==%lu", gni_mem_hdl->ref_count); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size = element_size; - reg_buf->payload = (uint64_t)buffer; - reg_buf->transport_private = (uint64_t)gni_mem_hdl; - - log_debug(nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - - if (gni_mem_hdl->ref_count==1) { - - gni_mem_hdl->extra=extra; - - if (ops == NNTI_BOP_RECV_QUEUE) { - nnti_gni_request_queue_handle_t *q_hdl=&transport_global_data.req_queue; - - q_hdl->reg_buf=reg_buf; - - server_req_queue_init( - q_hdl, - buffer, - element_size, - extra, - num_elements); - - reg_buf->payload_size=q_hdl->req_size; - - post_recv_queue_work_requests(reg_buf); - } else { - gni_mem_hdl->mem_hdl_list=&gni_mem_hdl->mem_hdl; - gni_mem_hdl->mem_hdl_count=1; - - gni_mem_hdl->mem_hdl=register_memory_segment(ops, buffer, element_size, extra); - } - } - - if ((ops & NNTI_BOP_REMOTE_WRITE) && (ops & NNTI_BOP_WITH_EVENTS)) { - post_recv_work_request(reg_buf); - } - - if (config.use_rdma_target_ack) { - if ((ops & NNTI_BOP_REMOTE_READ) || - (ops & NNTI_BOP_REMOTE_WRITE)) { - post_recv_work_request(reg_buf); - } - } - - if (rc==NNTI_OK) { - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(1, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=1; - assert(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val); - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id = NNTI_TRANSPORT_GEMINI; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword1 = gni_mem_hdl->mem_hdl.qword1; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword2 = gni_mem_hdl->mem_hdl.qword2; - - if (ops == NNTI_BOP_RECV_QUEUE) { - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size = transport_global_data.req_queue.req_size; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf = (uint64_t)transport_global_data.req_queue.req_buffer; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.type = NNTI_GNI_REQUEST_BUFFER; - } else { - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size = reg_buf->payload_size; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf = (uint64_t)reg_buf->payload; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.type = NNTI_GNI_SEND_SRC; - } - - if (gni_mem_hdl->ref_count==1) { - insert_buf_bufhash(reg_buf); - log_debug(nnti_debug_level, "reg_buf.buf.hash==%llu", - (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - log_debug(nnti_debug_level, "ref_count==1 called insert_buf_bufhash() (reg_buf=%p, reg_buf.hash6432=%llu)", - reg_buf, (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - NNTI_buffer_t *tmp_buf=get_buf_bufhash((uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - log_debug(nnti_debug_level, "immediate get_buf_bufhash() says tmp_buf=%p", tmp_buf); - } - } - -nthread_unlock(&nnti_mem_lock); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_gni_register_memory", reg_buf); - } - - log_debug(nnti_ee_debug_level, "exit (reg_buf=%p, reg_buf.hash6432=%llu)", reg_buf, (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - - return(rc); -} - -static NNTI_result_t unregister_memory(gni_mem_handle_t mem_hdl) -{ - int rc=GNI_RC_SUCCESS; /* return code */ - trios_declare_timer(call_time); - - log_debug(nnti_debug_level, "enter mem_hdl(%p)", mem_hdl); - - log_debug(nnti_debug_level, "unregister mem_hdl=(%llu,%llu)", (uint64_t)mem_hdl.qword1, (uint64_t)mem_hdl.qword2); - - trios_start_timer(call_time); - rc=GNI_MemDeregister (transport_global_data.nic_hdl, &mem_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "MemDeregister(mem_hdl) failed: %d", rc); - } - trios_stop_timer("buf deregister", call_time); - - switch(rc) { - case GNI_RC_SUCCESS: - rc=NNTI_OK; - default: - rc=NNTI_EIO; - } - - log_debug(nnti_debug_level, "exit mem_hdl(%p)", mem_hdl); - - return ((NNTI_result_t)rc); -} - -static int need_mem_cq(NNTI_buf_ops_t ops) -{ - int need_cq=0; - - log_debug(nnti_ee_debug_level, "enter"); - - if (!config.use_rdma_events) { - if ((ops == NNTI_BOP_RECV_QUEUE) || - (ops & NNTI_BOP_WITH_EVENTS)) { - need_cq=1; - } - } else { - if ((ops == NNTI_BOP_RECV_QUEUE) || - (ops & NNTI_BOP_WITH_EVENTS) || - (ops == NNTI_BOP_REMOTE_WRITE) || - ((ops == NNTI_BOP_REMOTE_READ) && (config.rdma_mode != RDMA_CROSSOVER))) { - need_cq=1; - } - } - - log_debug(nnti_ee_debug_level, "exit(%d)", need_cq); - - return(need_cq); -} - -static int need_wc_mem_cq(NNTI_buf_ops_t ops) -{ - int need_cq=0; - - log_debug(nnti_ee_debug_level, "enter"); - - if (!config.use_rdma_events) { - if ((ops == NNTI_BOP_RECV_QUEUE) || - (ops & NNTI_BOP_WITH_EVENTS)) { - need_cq=1; - } - } else { - if ((ops == NNTI_BOP_RECV_QUEUE) || - (ops & NNTI_BOP_WITH_EVENTS) || - (ops == NNTI_BOP_REMOTE_WRITE) || - (ops == NNTI_BOP_REMOTE_READ)) { - need_cq=1; - } - } - - log_debug(nnti_ee_debug_level, "exit(%d)", need_cq); - - return(need_cq); -} - -static nnti_gni_sge_t *decode_sge( - gni_cq_entry_t *ev_data, - gni_cq_handle_t cq_hdl) -{ - gni_return_t gni_rc=GNI_RC_SUCCESS; - - const NNTI_buffer_t *event_buf=NULL; - nnti_gni_sge_t *gni_sge =NULL; - nnti_gni_work_request_t *gni_wr =NULL; - - log_debug(nnti_debug_level, "enter"); - - uint64_t event_type = GNI_CQ_GET_TYPE(*ev_data); - if (event_type == GNI_CQ_EVENT_TYPE_POST) { - log_debug(nnti_debug_level, "ev_data.type == GNI_CQ_EVENT_TYPE_POST"); - } else if (event_type == GNI_CQ_EVENT_TYPE_SMSG) { - log_debug(nnti_debug_level, "ev_data.type == GNI_CQ_EVENT_TYPE_SMSG"); - } else if (event_type == GNI_CQ_EVENT_TYPE_MSGQ) { - log_debug(nnti_debug_level, "ev_data.type == GNI_CQ_EVENT_TYPE_MSGQ"); - } else { - log_debug(nnti_debug_level, "unknown type: ev_data.type == %llu", event_type); - } - - if (gni_cq_get_source(*ev_data) == 6) { - uint64_t event_inst_id = gni_cq_get_inst_id(*ev_data); - // this is a target buffer. determine if it is REQUEST_BUFFER or RECEIVE_BUFFER. - if (cq_hdl == transport_global_data.req_recv_mem_cq_hdl) { - log_debug(nnti_debug_level, "ev_data.source==6 and cq_hdl is transport_global_data.req_recv_cq_hdl, " - "so the event buffer is a REQUEST BUFFER."); - - int64_t next_req_index=nthread_counter_increment(&transport_global_data.req_queue.req_index); - next_req_index %= transport_global_data.req_queue.req_count; - - nnti_gni_memory_handle_t *gni_mem_hdl=GNI_MEM_HDL(transport_global_data.req_queue.reg_buf); - - gni_wr =gni_mem_hdl->wr_queue->at(next_req_index); - gni_sge=&gni_wr->sge_list[0]; - } else { - event_buf = get_buf_bufhash((uint32_t)event_inst_id); - if (event_buf == NULL) { - log_debug(nnti_debug_level, "unknown event -- ev_data.source==6 but inst_id is NOT a hash."); - } else { - log_debug(nnti_debug_level, "ev_data.source==6 and ev_data.inst_id is a hash, so the event buffer is a RECEIVE BUFFER."); - - nnti_gni_memory_handle_t *gni_mem_hdl=GNI_MEM_HDL(event_buf); - - gni_wr =gni_mem_hdl->wr_queue->front(); - gni_sge=&gni_wr->sge_list[0]; - } - } - } else { - gni_sge=get_sge_sgehash((uint32_t)gni_cq_get_inst_id(*ev_data)); - if (gni_sge==NULL) { - event_buf = get_buf_bufhash((uint32_t)gni_cq_get_inst_id(*ev_data)); - if (event_buf == NULL) { - log_error(nnti_debug_level, "ev_data.inst_id does NOT match either a bufhash or a sgehash. what happened? where did this event come from?"); - print_cq_event(ev_data, true); - - assert(gni_sge); - } else { - nnti_gni_memory_handle_t *gni_mem_hdl=GNI_MEM_HDL(event_buf); - - gni_wr =gni_mem_hdl->wr_queue->front(); - gni_sge=&gni_wr->sge_list[0]; - } - } - assert(gni_sge); - } - - log_debug(nnti_debug_level, "exit (gni_sge==%p)", gni_sge); - - return(gni_sge); -} - -static int cancel_wr( - nnti_gni_work_request_t *gni_wr) -{ - NNTI_result_t rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter (gni_wr=%p)", gni_wr); - - gni_wr->state=NNTI_GNI_WR_STATE_WAIT_COMPLETE; - gni_wr->nnti_wr->result=NNTI_ECANCELED; - - log_debug(nnti_debug_level, "exit (gni_wr==%p)", gni_wr); - - return(rc); -} - -static int process_event( - const NNTI_buffer_t *reg_buf, - nnti_gni_sge_t *gni_sge, - void *header, - gni_cq_handle_t cq_hdl, - gni_cq_entry_t *ev_data, - gni_post_descriptor_t *post_desc_ptr) -{ - int rc=NNTI_OK; - nnti_gni_work_request_t *gni_wr=NULL; - const NNTI_buffer_t *event_buf=NULL; - - log_level debug_level=nnti_debug_level; - - log_debug(debug_level, "enter (reg_buf=%p, gni_sge=%p, cq_hdl=%llu, ev_data=%llu)", reg_buf, gni_sge, (uint64_t)cq_hdl, (uint64_t)ev_data); - - if (!GNI_CQ_STATUS_OK(*ev_data)) { - if (GNI_CQ_OVERRUN(*ev_data)) { - log_debug(debug_level, "CQ overrun detected"); - } - return NNTI_EIO; - } - - assert(gni_sge); - - gni_wr=gni_sge->gni_wr; - assert(gni_wr); - - if ((gni_wr->nnti_wr) && (gni_wr->nnti_wr->ops == NNTI_BOP_ATOMICS)) { - gni_wr->state=NNTI_GNI_WR_STATE_RDMA_COMPLETE; - gni_wr->nnti_wr->result=NNTI_OK; - return NNTI_OK; - } - - log_debug(debug_level, "ev_data.inst_id==%llu, reg_buf.buf.hash==%llu", - (uint64_t)gni_cq_get_inst_id(*ev_data), - (uint64_t)hash6432shift(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - - event_buf=reg_buf; - - log_debug(debug_level, "event_buf=%p; gni_sge=%p; gni_wr=%p; gni_wr->last_op=%lld; gni_wr.hash==%llu", - event_buf, gni_sge, gni_wr, (int64_t)gni_wr->last_op, (uint64_t)hash6432shift((uint64_t)gni_wr)); - - debug_level=nnti_debug_level; - if (gni_wr->last_op==GNI_OP_SEND_REQUEST) { - if (gni_sge->state==NNTI_GNI_SGE_STATE_STARTED) { - - log_debug(debug_level, "SEND request event - event_buf==%p, state==%d", event_buf, gni_sge->state); - - log_debug(debug_level, "SEND request completion - event_buf==%p", event_buf); - gni_sge->state=NNTI_GNI_SGE_STATE_COMPLETE; - - gni_wr->wc->byte_offset=gni_wr->wc->src_offset; - } - } else if (gni_wr->last_op==GNI_OP_SEND_BUFFER) { - if (gni_sge->state==NNTI_GNI_SGE_STATE_STARTED) { - log_debug(debug_level, "SEND buffer event - event_buf==%p, state==%d", event_buf, gni_sge->state); - - if (post_desc_ptr == &gni_sge->post_desc) { - log_debug(debug_level, "SEND buffer completion - event_buf==%p", event_buf); - gni_sge->state=NNTI_GNI_SGE_STATE_COMPLETE; - } else { - log_debug(debug_level, "SEND buffer - unknown post descriptor - post_desc_ptr(%p) != &gni_sge->post_desc(%p)", - post_desc_ptr, &gni_sge->post_desc); - } - gni_wr->wc->byte_offset=gni_wr->wc->src_offset; - } - } else if (gni_wr->last_op==GNI_OP_PUT_INITIATOR) { - if (gni_sge->state==NNTI_GNI_SGE_STATE_STARTED) { - log_debug(debug_level, "RDMA write event - event_buf==%p, gni_sge->state==%d", event_buf, gni_sge->state); - if (post_desc_ptr == &gni_sge->post_desc) { - log_debug(debug_level, "RDMA write (initiator) completion - event_buf==%p", event_buf); - gni_sge->state=NNTI_GNI_SGE_STATE_COMPLETE; - } else { - log_debug(debug_level, "RDMA write - unknown post descriptor - post_desc_ptr(%p) != &gni_wr->post_desc(%p)", - post_desc_ptr, &gni_sge->post_desc); - } - gni_wr->wc->byte_offset=gni_wr->wc->src_offset; - } - } else if (gni_wr->last_op == GNI_OP_GET_INITIATOR) { - if (gni_sge->state==NNTI_GNI_SGE_STATE_STARTED) { - log_debug(debug_level, "RDMA read event - event_buf==%p, state==%d", event_buf, gni_sge->state); - if (post_desc_ptr == &gni_sge->post_desc) { - log_debug(debug_level, "RDMA read (initiator) completion - event_buf==%p", event_buf); - gni_sge->state=NNTI_GNI_SGE_STATE_COMPLETE; - } else { - log_debug(debug_level, "RDMA read - unknown post descriptor - post_desc_ptr(%p) != &gni_wr->post_desc(%p)", - post_desc_ptr, &gni_sge->post_desc); - } - gni_wr->wc->byte_offset=gni_wr->wc->dest_offset; - } - } else if (gni_wr->last_op == GNI_OP_NEW_REQUEST) { - int64_t req_count=0; - - nnti_gni_mbox_backlog_t *backlog_element=(nnti_gni_mbox_backlog_t*)malloc(sizeof(nnti_gni_mbox_backlog_t)); - - backlog_element->conn=get_conn_instance((uint32_t)gni_cq_get_inst_id(*ev_data)); - assert(backlog_element->conn); - backlog_element->queue_index=gni_wr->index; - backlog_element->mbox_copy_from=header; - - nthread_lock(&nnti_mbox_backlog_lock); - mbox_backlog.push_back(backlog_element); - nthread_unlock(&nnti_mbox_backlog_lock); - - execute_mbox_backlog(); - - } else if (gni_wr->last_op == GNI_OP_RECEIVE) { - log_debug(debug_level, "RDMA write event - event_buf==%p, state==%d", event_buf, gni_wr->state); - if (config.use_rdma_events) { -// if ((gni_wr->op_state.rdma_init ==true) && -// (gni_wr->op_state.rdma_complete==false)) { -// log_debug(debug_level, "RDMA write (receive buffer) completion - event_buf==%p", event_buf); -// gni_wr->op_state.rdma_complete=true; -// } else if ((gni_wr->op_state.rdma_init ==true) && -// (gni_wr->op_state.rdma_complete==true) && -// (gni_wr->op_state.wc_complete ==false)) { -// log_debug(debug_level, "RDMA write ACK (receive buffer) completion - event_buf==%p", event_buf); -// gni_wr->op_state.wc_complete=true; -// gni_wr->wc->byte_offset=gni_wr->wc->dest_offset; -// } - } else { - log_debug(debug_level, "RDMA write ACK (receive buffer) completion - event_buf==%p", event_buf); - gni_wr->state=NNTI_GNI_WR_STATE_RDMA_COMPLETE; - gni_wr->wc->byte_offset=gni_wr->wc->dest_offset; - } - } else if (gni_wr->last_op == GNI_OP_GET_TARGET) { - log_error(nnti_debug_level, "GNI_OP_GET_TARGET is an invalid op - gni_wr->last_op(%d)", gni_wr->last_op); - abort(); - } else if (gni_wr->last_op == GNI_OP_PUT_TARGET) { - log_error(nnti_debug_level, "GNI_OP_PUT_TARGET is an invalid op - gni_wr->last_op(%d)", gni_wr->last_op); - abort(); - } else { - log_error(nnti_debug_level, "unknown gni_wr->last_op(%d)", gni_wr->last_op); - abort(); - } - - print_wc(gni_wr->wc); - - log_debug(nnti_ee_debug_level, "exit"); - return (rc); -} - -static NNTI_result_t execute_mbox_copy(nnti_gni_mbox_backlog_t *backlog_element) -{ - gni_return_t gni_rc=GNI_RC_SUCCESS; - int64_t mbox_offset=0; - - nnti_gni_request_queue_handle_t *q=&transport_global_data.req_queue; - nnti_gni_memory_handle_t *gni_mem_hdl=GNI_MEM_HDL(transport_global_data.req_queue.reg_buf); - nnti_gni_work_request_t *gni_wr=gni_mem_hdl->wr_queue->at(backlog_element->queue_index); - - if ((gni_wr->state != NNTI_GNI_WR_STATE_POSTED) && (gni_wr->state != NNTI_GNI_WR_STATE_ATTACHED)) { - return NNTI_EAGAIN; - } - gni_wr->last_op=GNI_OP_NEW_REQUEST; - - int64_t req_buffer_offset=GNI_ELEMENT_OFFSET(q->reg_buf, gni_wr->index); - - log_debug(nnti_debug_level, "&q->req_buffer[req_buffer_offset=%lld] = %p, mbox req_addr = %x", - req_buffer_offset, &q->req_buffer[req_buffer_offset], backlog_element->mbox_copy_from); - - memcpy(&q->req_buffer[req_buffer_offset], backlog_element->mbox_copy_from, q->req_size); - - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_SmsgRelease(backlog_element->conn->recv_mbox->ep_hdl); - nthread_unlock(&nnti_gni_lock); - if (gni_rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "SmsgRelease(request) failed: %d", gni_rc); - - send_back_credits(backlog_element->conn, 1); - - log_debug(nnti_debug_level, "recv completion - gni_wr->index=%llu", gni_wr->index); - - gni_wr->wc->ack_received=1; - gni_wr->wc->inst_id =backlog_element->conn->peer_instance; - gni_wr->wc->op =GNI_OP_SEND_REQUEST; - gni_wr->wc->byte_len =q->req_size; - gni_wr->wc->byte_offset =req_buffer_offset; - gni_wr->wc->src_offset =0; - gni_wr->wc->dest_offset =req_buffer_offset; - - gni_wr->state=NNTI_GNI_WR_STATE_RDMA_COMPLETE; - - return NNTI_OK; - -} -static NNTI_result_t execute_mbox_backlog(void) -{ - NNTI_result_t rc=NNTI_OK; - - mbox_backlog_iter_t backlog_iter; - - nthread_lock(&nnti_mbox_backlog_lock); - log_debug(nnti_debug_level, "mbox_backlog.size()=%d", mbox_backlog.size()); - - backlog_iter=mbox_backlog.begin(); - while (backlog_iter != mbox_backlog.end()) { - nnti_gni_mbox_backlog_t *backlog_element=*backlog_iter; - - rc=execute_mbox_copy(backlog_element); - if (rc == NNTI_OK) { - log_debug(nnti_debug_level, "execute_mbox_copy() success: %d", rc); - - backlog_iter = mbox_backlog.erase(backlog_iter); - - } else if (rc == NNTI_EAGAIN) { - log_warn(nnti_debug_level, "execute_mbox_copy() receive queue is full: %d", rc); - - break; - - } else { - log_warn(nnti_debug_level, "execute_mbox_copy() unknown error: %d", rc); - - break; - } - } - - nthread_unlock(&nnti_mbox_backlog_lock); - - return rc; -} - -static NNTI_result_t post_recv_queue_work_requests( - NNTI_buffer_t *reg_buf) -{ - nnti_gni_work_request_t *gni_wr=NULL; - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - nnti_gni_request_queue_handle_t *q=&transport_global_data.req_queue; - - log_debug(nnti_ee_debug_level, "enter (reg_buf=%p)", reg_buf); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)reg_buf->transport_private; - assert(gni_mem_hdl); - - for (int64_t index=0;index < q->req_count;index++) { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - assert(gni_wr); - - gni_wr->sge_list=&gni_wr->sge; - gni_wr->sge_count=1; - gni_wr->sge_list[0].gni_wr=gni_wr; - - gni_wr->reg_buf=reg_buf; - gni_wr->last_op=GNI_OP_NEW_REQUEST; - gni_wr->state =NNTI_GNI_WR_STATE_POSTED; - - gni_wr->index = index; - gni_wr->wc = GNI_WC_ADDRESS(reg_buf, index); - - gni_mem_hdl->wr_queue->push_back(gni_wr); - } - - log_debug(nnti_ee_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static NNTI_result_t post_recv_work_request( - NNTI_buffer_t *reg_buf) -{ - nnti_gni_work_request_t *gni_wr=NULL; - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - log_debug(nnti_ee_debug_level, "enter (reg_buf=%p)", reg_buf); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)reg_buf->transport_private; - assert(gni_mem_hdl); - - if (config.use_wr_pool) { - if (need_wc_mem_cq(reg_buf->ops) == 1) { - gni_wr=wr_pool_target_pop(); - } else { - gni_wr=wr_pool_initiator_pop(); - } - } else { - gni_wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(gni_wr, 0, sizeof(nnti_gni_work_request_t)); - nthread_lock_init(&gni_wr->lock); - } - assert(gni_wr); - - gni_wr->sge_list=&gni_wr->sge; - gni_wr->sge_count=1; - gni_wr->sge_list[0].gni_wr=gni_wr; - - gni_wr->reg_buf=reg_buf; - gni_wr->last_op=GNI_OP_RECEIVE; - gni_wr->state =NNTI_GNI_WR_STATE_POSTED; - - gni_wr->wc = GNI_WC_ADDRESS(reg_buf, 0); - - nthread_lock(&gni_mem_hdl->wr_queue_lock); - gni_mem_hdl->wr_queue->push_back(gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - log_debug(nnti_ee_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static NNTI_result_t repost_recv_work_request( - NNTI_buffer_t *reg_buf, - nnti_gni_work_request_t *gni_wr) -{ - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - log_debug(nnti_ee_debug_level, "enter (reg_buf=%p)", reg_buf); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)reg_buf->transport_private; - assert(gni_mem_hdl); - - gni_wr->state=NNTI_GNI_WR_STATE_POSTED; - - gni_mem_hdl->wr_queue->push_back(gni_wr); - - log_debug(nnti_ee_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static int8_t is_wr_canceling( - nnti_gni_work_request_t *gni_wr) -{ - int8_t rc=FALSE; - - if (gni_wr->state==NNTI_GNI_WR_STATE_CANCELING) { - rc=TRUE; - } - - log_debug(nnti_debug_level, "exit (rc=%d)", rc); - - return(rc); -} - -static int8_t is_wr_complete( - nnti_gni_work_request_t *gni_wr) -{ - int8_t rc=FALSE; - - if ((gni_wr->state==NNTI_GNI_WR_STATE_RDMA_COMPLETE) || - (gni_wr->state==NNTI_GNI_WR_STATE_WAIT_COMPLETE)) { - rc=TRUE; - } else { - rc=TRUE; /* assume complete. flip to incomplete if any SGE is not complete. */ - for (uint32_t i=0;isge_count;i++) { - if (gni_wr->sge_list[i].state != NNTI_GNI_SGE_STATE_COMPLETE) { - rc=FALSE; - break; - } - } - if (rc==TRUE) { - // all SGEs are complete, so the WR is too. - gni_wr->state=NNTI_GNI_WR_STATE_RDMA_COMPLETE; - } - } - - log_debug(nnti_ee_debug_level, "exit (rc=%d)", rc); - return(rc); -} - -static int8_t is_wr_complete( - NNTI_work_request_t *wr) -{ - nnti_gni_work_request_t *gni_wr=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - gni_wr=GNI_WORK_REQUEST(wr); - assert(gni_wr); - - return(is_wr_complete(gni_wr)); -} - -static int8_t is_any_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - uint32_t *which) -{ - int8_t rc=FALSE; - - log_debug(nnti_debug_level, "enter"); - - for (uint32_t i=0;iop = wr->ops; - if (nnti_rc == NNTI_OK) { - status->result = wr->result; - } else { - status->result = nnti_rc; - } - if (status->result==NNTI_OK) { - if (gni_wr->peer_instance) { - conn = get_conn_instance(gni_wr->peer_instance); - } else { - conn = get_conn_instance(gni_wr->wc->inst_id); - } - assert(conn); - - if (gni_wr->nnti_wr->ops != NNTI_BOP_ATOMICS) { - status->start = (uint64_t)gni_wr->reg_buf->payload; - status->offset = gni_wr->wc->byte_offset; - status->length = gni_wr->wc->byte_len; - } - switch (gni_wr->last_op) { - case GNI_OP_PUT_INITIATOR: - case GNI_OP_GET_TARGET: - case GNI_OP_SEND_REQUEST: - case GNI_OP_SEND_BUFFER: - copy_peer( - &status->src, - &transport_global_data.me); - copy_peer( - &status->dest, - &conn->peer); - break; - case GNI_OP_GET_INITIATOR: - case GNI_OP_PUT_TARGET: - case GNI_OP_NEW_REQUEST: - case GNI_OP_RECEIVE: - copy_peer( - &status->src, - &conn->peer); - copy_peer( - &status->dest, - &transport_global_data.me); - break; - } - } -} - -static void create_peer(NNTI_peer_t *peer, char *name, NNTI_ip_addr addr, NNTI_tcp_port port, uint32_t ptag, uint32_t cookie, NNTI_instance_id instance) -{ - log_debug(nnti_ee_debug_level, "enter"); - - sprintf(peer->url, "gni://%s:%u/?ptag=%lu&cookie=%lu", name, ntohs(port), (uint64_t)ptag, (uint64_t)cookie); - - peer->peer.transport_id =NNTI_TRANSPORT_GEMINI; - peer->peer.NNTI_remote_process_t_u.gni.addr =addr; - peer->peer.NNTI_remote_process_t_u.gni.port =port; - peer->peer.NNTI_remote_process_t_u.gni.inst_id=instance; - - log_debug(nnti_ee_debug_level, "exit"); -} - -static void copy_peer(NNTI_peer_t *dst_peer, NNTI_peer_t *src_peer) -{ - log_debug(nnti_ee_debug_level, "enter"); - - strcpy(dst_peer->url, src_peer->url); - - dst_peer->peer.transport_id =src_peer->peer.transport_id; - dst_peer->peer.NNTI_remote_process_t_u.gni.addr =src_peer->peer.NNTI_remote_process_t_u.gni.addr; - dst_peer->peer.NNTI_remote_process_t_u.gni.port =src_peer->peer.NNTI_remote_process_t_u.gni.port; - dst_peer->peer.NNTI_remote_process_t_u.gni.inst_id=src_peer->peer.NNTI_remote_process_t_u.gni.inst_id; - - log_debug(nnti_ee_debug_level, "exit"); -} - -static NNTI_result_t insert_conn_peer(const NNTI_peer_t *peer, nnti_gni_connection_t *conn) -{ - NNTI_result_t rc=NNTI_OK; - addrport_key key; - - key.addr = peer->peer.NNTI_remote_process_t_u.gni.addr; - key.port = peer->peer.NNTI_remote_process_t_u.gni.port; - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "insert_conn_peer", peer); - } - - if (nthread_lock(&nnti_conn_peer_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (connections_by_peer.find(key) != connections_by_peer.end()) { - print_peer_map(); - assert(connections_by_peer.find(key) == connections_by_peer.end()); - } - connections_by_peer[key] = conn; // add to connection map - nthread_unlock(&nnti_conn_peer_lock); - - log_debug(nnti_debug_level, "peer connection added (conn=%p)", conn); - - return(rc); -} -static NNTI_result_t insert_conn_instance(const NNTI_instance_id instance, nnti_gni_connection_t *conn) -{ - NNTI_result_t rc=NNTI_OK; - - if (nthread_lock(&nnti_conn_instance_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (connections_by_instance.find(instance) != connections_by_instance.end()) { - print_instance_map(); - assert(connections_by_instance.find(instance) == connections_by_instance.end()); - } - connections_by_instance[instance] = conn; - nthread_unlock(&nnti_conn_instance_lock); - - log_debug(nnti_debug_level, "instance connection added (conn=%p)", conn); - - return(rc); -} -static nnti_gni_connection_t *get_conn_peer(const NNTI_peer_t *peer) -{ - nnti_gni_connection_t *conn = NULL; - addrport_key key; - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "looking for peer", - "get_conn_peer", peer); - } - - memset(&key, 0, sizeof(addrport_key)); - key.addr=peer->peer.NNTI_remote_process_t_u.gni.addr; - key.port=peer->peer.NNTI_remote_process_t_u.gni.port; - - if (nthread_lock(&nnti_conn_peer_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (connections_by_peer.find(key) != connections_by_peer.end()) { - conn = connections_by_peer[key]; - } - nthread_unlock(&nnti_conn_peer_lock); - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - return conn; - } - - log_debug(nnti_debug_level, "connection NOT found"); - print_peer_map(); - - return(NULL); -} -static nnti_gni_connection_t *get_conn_instance(const NNTI_instance_id instance) -{ - nnti_gni_connection_t *conn=NULL; - - log_debug(nnti_debug_level, "looking for instance=%llu", (unsigned long long)instance); - if (nthread_lock(&nnti_conn_instance_lock)) log_warn(nnti_debug_level, "failed to get thread lock");; - if (connections_by_instance.find(instance) != connections_by_instance.end()) { - conn = connections_by_instance[instance]; - } - nthread_unlock(&nnti_conn_instance_lock); - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - return conn; - } - - log_debug(nnti_debug_level, "connection NOT found (instance==%llu)", (uint64_t)instance); - print_instance_map(); - - return(NULL); -} -static NNTI_peer_t *get_peer_by_url(const char *url) -{ - nnti_gni_connection_t *conn = NULL; - - log_debug(nnti_debug_level, "looking for url=%s", url); - - conn_by_peer_iter_t i; - if (nthread_lock(&nnti_conn_peer_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - for (i=connections_by_peer.begin(); i != connections_by_peer.end(); i++) { - log_debug(nnti_debug_level, "peer_map key=(%llu,%llu) conn=%p, url=%s", - (uint64_t)i->first.addr, (uint64_t)i->first.port, i->second, i->second->peer.url); - if (strcmp(i->second->peer.url, url) == 0) { - conn=i->second; - break; - } - } - nthread_unlock(&nnti_conn_peer_lock); - - if (conn != NULL) { - log_debug(nnti_debug_level, "peer found"); - NNTI_peer_t *peer=(NNTI_peer_t*)malloc(sizeof(NNTI_peer_t)); - *peer=conn->peer; - return peer; - } - - return(NULL); -} -static nnti_gni_connection_t *del_conn_peer(const NNTI_peer_t *peer) -{ - nnti_gni_connection_t *conn=NULL; - addrport_key key; - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "get_conn_peer", peer); - } - - memset(&key, 0, sizeof(addrport_key)); - key.addr=peer->peer.NNTI_remote_process_t_u.gni.addr; - key.port=peer->peer.NNTI_remote_process_t_u.gni.port; - - if (nthread_lock(&nnti_conn_peer_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (connections_by_peer.find(key) != connections_by_peer.end()) { - conn = connections_by_peer[key]; - } - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - connections_by_peer.erase(key); - del_conn_instance(conn->peer_instance); - } else { - log_debug(nnti_debug_level, "connection NOT found"); - } - nthread_unlock(&nnti_conn_peer_lock); - - return(conn); -} -static nnti_gni_connection_t *del_conn_instance(const NNTI_instance_id instance) -{ - nnti_gni_connection_t *conn=NULL; - log_level debug_level = nnti_debug_level; - - if (nthread_lock(&nnti_conn_instance_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (connections_by_instance.find(instance) != connections_by_instance.end()) { - conn = connections_by_instance[instance]; - } - - if (conn != NULL) { - log_debug(debug_level, "connection found"); - connections_by_instance.erase(instance); - } else { - log_debug(debug_level, "connection NOT found"); - } - nthread_unlock(&nnti_conn_instance_lock); - - return(conn); -} -static void print_peer_map() -{ - if (!logging_debug(nnti_debug_level)) { - return; - } - - conn_by_peer_iter_t i; - for (i=connections_by_peer.begin(); i != connections_by_peer.end(); i++) { - log_debug(nnti_debug_level, "peer_map key=(%llu,%llu) conn=%p", - (uint64_t)i->first.addr, (uint64_t)i->first.port, i->second); - } -} -static void print_instance_map() -{ - if (!logging_debug(nnti_debug_level)) { - return; - } - - conn_by_inst_iter_t i; - for (i=connections_by_instance.begin(); i != connections_by_instance.end(); i++) { - log_debug(nnti_debug_level, "instance_map key=%llu conn=%p", i->first, i->second); - } -} - - -static NNTI_result_t insert_buf_bufhash(NNTI_buffer_t *buf) -{ - NNTI_result_t rc=NNTI_OK; - uint32_t h=hash6432shift((uint64_t)buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf); - - if (nthread_lock(&nnti_buf_bufhash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - - log_debug(nnti_debug_level, "adding buf=%p ; bufhash=%llu", buf, (uint64_t)h); - - assert(buffers_by_bufhash.find(h) == buffers_by_bufhash.end()); - buffers_by_bufhash[h] = buf; - - log_debug(nnti_debug_level, "bufhash buffer added (buf=%p ; buf.hash=%llu)", buf, (uint64_t)h); - - nthread_unlock(&nnti_buf_bufhash_lock); - - print_bufhash_map(); - - return(rc); -} -static NNTI_buffer_t *get_buf_bufhash(const uint32_t bufhash) -{ - NNTI_buffer_t *buf=NULL; - - if (nthread_lock(&nnti_buf_bufhash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - - log_debug(nnti_debug_level, "looking for bufhash=%llu", (uint64_t)bufhash); - - if (buffers_by_bufhash.find(bufhash) != buffers_by_bufhash.end()) { - buf = buffers_by_bufhash[bufhash]; - } - - log_debug(nnti_debug_level, "found buffer (buf=%p ; buf.hash=%llu)", buf, (uint64_t)bufhash); - - nthread_unlock(&nnti_buf_bufhash_lock); - - if (buf != NULL) { - log_debug(nnti_debug_level, "buffer found (buf=%p ; buf.hash=%llu)", buf, (uint64_t)bufhash); - return buf; - } - - log_debug(nnti_debug_level, "buffer NOT found"); - - print_bufhash_map(); - - return(NULL); -} -static NNTI_buffer_t *del_buf_bufhash(NNTI_buffer_t *victim) -{ - NNTI_buffer_t *buf=NULL; - uint32_t h=hash6432shift((uint64_t)victim->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf); - log_level debug_level = nnti_debug_level; - - if (nthread_lock(&nnti_buf_bufhash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - - log_debug(debug_level, "deleting (victim=%p ; bufhash=%llu)", victim, (uint64_t)h); - - if (buffers_by_bufhash.find(h) != buffers_by_bufhash.end()) { - buf = buffers_by_bufhash[h]; - } - - if (buf != NULL) { - log_debug(debug_level, "buffer found and deleted (victim=%p ; buf=%p ; bufhash=%llu)", victim, buf, (uint64_t)h); - buffers_by_bufhash.erase(h); - } else { - log_debug(debug_level, "buffer NOT found"); - } - - log_debug(debug_level, "deleted (victim=%p ; buf=%p ; bufhash=%llu)", victim, buf, (uint64_t)h); - - nthread_unlock(&nnti_buf_bufhash_lock); - - print_bufhash_map(); - - return(buf); -} -static void print_bufhash_map() -{ - log_level debug_level=nnti_debug_level; - - if (!logging_debug(debug_level)) { - return; - } - - buf_by_bufhash_iter_t i; - if (nthread_lock(&nnti_buf_bufhash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - for (i=buffers_by_bufhash.begin(); i != buffers_by_bufhash.end(); i++) { - log_debug(debug_level, "bufhash_map key=%llu buf=%p", i->first, i->second); - } - nthread_unlock(&nnti_buf_bufhash_lock); -} - -static NNTI_result_t insert_sge_sgehash(nnti_gni_sge_t *sge) -{ - NNTI_result_t rc=NNTI_OK; - uint32_t h=hash6432shift((uint64_t)sge); - - if (nthread_lock(&nnti_sge_sgehash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - - log_debug(nnti_debug_level, "adding sgehash (sge=%p ; sge.hash=%llu)", sge, (uint64_t)h); - - assert(sge_by_sgehash.find(h) == sge_by_sgehash.end()); - sge_by_sgehash[h] = sge; - - log_debug(nnti_debug_level, "added sgehash (sge=%p ; sge.hash=%llu)", sge, (uint64_t)h); - - nthread_unlock(&nnti_sge_sgehash_lock); - - return(rc); - -} -static nnti_gni_sge_t *get_sge_sgehash(const uint32_t sgehash) -{ - nnti_gni_sge_t *sge=NULL; - - if (nthread_lock(&nnti_sge_sgehash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - - log_debug(nnti_debug_level, "looking for sgehash=%llu", (uint64_t)sgehash); - - if (sge_by_sgehash.find(sgehash) != sge_by_sgehash.end()) { - sge = sge_by_sgehash[sgehash]; - } - - log_debug(nnti_debug_level, "found sge (sge=%p ; sgehash=%llu)", sge, (uint64_t)sgehash); - - nthread_unlock(&nnti_sge_sgehash_lock); - - if (sge != NULL) { - log_debug(nnti_debug_level, "sge found (sge=%p ; sgehash=%llu)", sge, (uint64_t)sgehash); - return sge; - } - - log_debug(nnti_debug_level, "sge NOT found"); - - print_sgehash_map(); - - return(NULL); -} -static nnti_gni_sge_t *del_sge_sgehash(nnti_gni_sge_t *victim) -{ - nnti_gni_sge_t *sge=NULL; - - uint32_t h=hash6432shift((uint64_t)victim); - log_level debug_level = nnti_debug_level; - - if (nthread_lock(&nnti_sge_sgehash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - - log_debug(debug_level, "deleting sgehash=%llu", (uint64_t)h); - - if (sge_by_sgehash.find(h) != sge_by_sgehash.end()) { - sge = sge_by_sgehash[h]; - } - - if (sge != NULL) { - log_debug(nnti_debug_level, "sge found and deleted (victim=%p ; sge=%p ; sgehash=%llu)", victim, sge, (uint64_t)h); - sge_by_sgehash.erase(h); - } else { - log_debug(debug_level, "sge NOT found"); - } - - log_debug(debug_level, "deleted (sge=%p ; sgehash=%llu)", sge, (uint64_t)h); - - nthread_unlock(&nnti_sge_sgehash_lock); - - print_sgehash_map(); - - return(sge); -} -static void print_sgehash_map() -{ - log_level debug_level=nnti_debug_level; - - if (!logging_debug(debug_level)) { - return; - } - - sge_by_sgehash_iter_t i; - if (nthread_lock(&nnti_sge_sgehash_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - for (i=sge_by_sgehash.begin(); i != sge_by_sgehash.end(); i++) { - log_debug(debug_level, "sgehash_map key=%llu sge=%p", i->first, i->second); - } - nthread_unlock(&nnti_sge_sgehash_lock); -} - -static NNTI_result_t wr_pool_register( - nnti_gni_work_request_t *wr, - gni_cq_handle_t cq_hdl) -{ -// int rc=GNI_RC_SUCCESS; - - log_debug(nnti_debug_level, "enter"); - -// rc=GNI_MemRegister (transport_global_data.nic_hdl, -// (uint64_t)&wr->wc, -// sizeof(nnti_gni_work_completion_t), -// cq_hdl, -// GNI_MEM_READWRITE, -// (uint32_t)-1, -// &wr->wc_mem_hdl); -// if (rc!=GNI_RC_SUCCESS) { -// log_error(nnti_debug_level, "MemRegister(wc_mem_hdl) failed: rc=%d, %s", rc, strerror(errno)); -// return(NNTI_EIO); -// } -// -// wr->wc_registered=FALSE; - - log_debug(nnti_debug_level, "exit"); - - return(NNTI_OK); -} -static NNTI_result_t wr_pool_deregister( - nnti_gni_work_request_t *wr) -{ -// int rc=GNI_RC_SUCCESS; - - log_debug(nnti_debug_level, "enter"); - -// if (wr->wc_registered==FALSE) { -// log_debug(nnti_debug_level, "exit wr(%p) - not registered", wr); -// return(NNTI_OK); -// } -// -// rc=GNI_MemDeregister(transport_global_data.nic_hdl, &wr->wc_mem_hdl); -// if (rc!=GNI_RC_SUCCESS) { -// log_error(nnti_debug_level, "MemDeregister(wc_mem_hdl) failed: rc=%d", rc); -// return(NNTI_EIO); -// } -// -// wr->wc_registered=FALSE; - - log_debug(nnti_debug_level, "exit"); - - return(NNTI_OK); -} -static NNTI_result_t wr_pool_init(void) -{ - NNTI_result_t rc=NNTI_OK; - uint32_t i; - nnti_gni_work_request_t *wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - for (i=0;ilock); - rc=wr_pool_register(wr, transport_global_data.mem_cq_hdl); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "failed to register target work request: rc=%d", rc); - goto cleanup; - } - wr->is_initiator=FALSE; - wr_pool_target_push(wr); - - wr=(nnti_gni_work_request_t *)malloc(sizeof(nnti_gni_work_request_t)); - memset(wr, 0, sizeof(nnti_gni_work_request_t)); - assert(wr); - nthread_lock_init(&wr->lock); - rc=wr_pool_register(wr, NULL); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "failed to register initiator work request: rc=%d", rc); - goto cleanup; - } - wr->is_initiator=TRUE; - wr_pool_initiator_push(wr); - } - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(rc); -} -static nnti_gni_work_request_t *wr_pool_target_pop(void) -{ - nnti_gni_work_request_t *wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - if (nthread_lock(&nnti_wr_pool_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (!target_wr_pool.empty()) { - wr=target_wr_pool.front(); - target_wr_pool.pop_front(); - } - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return(wr); -} -static nnti_gni_work_request_t *wr_pool_initiator_pop(void) -{ - nnti_gni_work_request_t *wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - if (nthread_lock(&nnti_wr_pool_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - if (!initiator_wr_pool.empty()) { - wr=initiator_wr_pool.front(); - initiator_wr_pool.pop_front(); - } - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return(wr); -} -static void wr_pool_target_push(nnti_gni_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter"); - - memset(&wr->wc, 0, sizeof(nnti_gni_work_completion_t)); - wr->state =NNTI_GNI_WR_STATE_RESET; - wr->last_op =0; - - if (nthread_lock(&nnti_wr_pool_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - target_wr_pool.push_front(wr); - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return; -} -static void wr_pool_initiator_push(nnti_gni_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter"); - - memset(&wr->wc, 0, sizeof(nnti_gni_work_completion_t)); - wr->state =NNTI_GNI_WR_STATE_RESET; - wr->last_op =0; - - if (nthread_lock(&nnti_wr_pool_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - initiator_wr_pool.push_front(wr); - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return; -} -static NNTI_result_t wr_pool_fini(void) -{ - NNTI_result_t rc=NNTI_OK; - nnti_gni_work_request_t *wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - if (nthread_lock(&nnti_wr_pool_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - while (!target_wr_pool.empty()) { - wr=target_wr_pool.front(); - target_wr_pool.pop_front(); - assert(wr); - rc=wr_pool_deregister(wr); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "failed to deregister target work request: rc=%d", rc); - goto cleanup; - } - free(wr); - } - while (!initiator_wr_pool.empty()) { - wr=initiator_wr_pool.front(); - initiator_wr_pool.pop_front(); - assert(wr); - rc=wr_pool_deregister(wr); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "failed to deregister initiator work request: rc=%d", rc); - goto cleanup; - } - free(wr); - } - -cleanup: - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - -/* - * Try hard to read the whole buffer. Abort on read error. - */ -static int tcp_read(int sock, void *incoming, size_t len) -{ - int bytes_this_read=0; - int bytes_left=len; - int bytes_read=0; - - while (bytes_left > 0) { - bytes_this_read = read(sock, (char *)incoming + bytes_read, bytes_left); - if (bytes_this_read < 0) { - return bytes_this_read; - } - if (bytes_this_read == 0) { - break; - } - bytes_left -= bytes_this_read; - bytes_read += bytes_this_read; - } - return bytes_read; -} - -/* - * Try hard to write the whole buffer. Abort on write error. - */ -static int tcp_write(int sock, const void *outgoing, size_t len) -{ - int bytes_this_write=0; - int bytes_left=len; - int bytes_written=0; - - while (bytes_left > 0) { - bytes_this_write = write(sock, (const char *)outgoing + bytes_written, bytes_left); - if (bytes_this_write < 0) { - return bytes_this_write; - } - bytes_left -= bytes_this_write; - bytes_written += bytes_this_write; - } - return bytes_written; -} - -/* - * Two processes exchange data over a TCP socket. Both sides send and receive the - * same amount of data. Only one process can declare itself the server (is_server!=0), - * otherwise this will hang because both will wait for the read to complete. - * - * Server receives, then sends. - * Client sends, then receives. - */ -static int tcp_exchange(int sock, int is_server, void *incoming, void *outgoing, size_t len) -{ - int rc=0; - - if (is_server) { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_read(sock, incoming, len); - trios_stop_timer("tcp_read", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "server failed to read GNI connection info: errno=%d", errno); - goto out; - } - if (rc != (int) len) { - log_error(nnti_debug_level, "partial read, %d/%d bytes", rc, (int) len); - rc = 1; - goto out; - } - } else { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_write(sock, outgoing, len); - trios_stop_timer("tcp_write", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "client failed to write GNI connection info: errno=%d", errno); - goto out; - } - } - - if (is_server) { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_write(sock, outgoing, len); - trios_stop_timer("tcp_write", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "server failed to write GNI connection info: errno=%d", errno); - goto out; - } - } else { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_read(sock, incoming, len); - trios_stop_timer("tcp_read", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "client failed to read GNI connection info: errno=%d", errno); - goto out; - } - if (rc != (int) len) { - log_error(nnti_debug_level, "partial read, %d/%d bytes", rc, (int) len); - rc = 1; - goto out; - } - } - - rc = 0; - -out: - return rc; -} - -static void transition_connection_to_ready( - int sock, - int is_server, - nnti_gni_connection_t *conn) -{ - int rc=NNTI_OK; - trios_declare_timer(callTime); - - trios_start_timer(callTime); - /* final sychronization to ensure both sides have posted RTRs */ - rc = tcp_exchange(sock, is_server, &rc, &rc, sizeof(rc)); - trios_stop_timer("transition tcp_exchange", callTime); -} - -static int build_send_mbox( - nnti_gni_connection_t *c, - uint32_t msg_maxsize) -{ - int rc; - - nnti_gni_request_queue_handle_t *q=&transport_global_data.req_queue; - - const unsigned int CACHELINE_SIZE=64; - - unsigned int bytes_per_mbox=0; - unsigned int adjusted_bytes_per_mbox=0; - - gni_smsg_attr_t smsg_attributes; - - - smsg_attributes.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT; - smsg_attributes.mbox_maxcredit = config.queue_elements_per_connection; - smsg_attributes.msg_maxsize = msg_maxsize; - - rc=GNI_SmsgBufferSizeNeeded(&smsg_attributes, &bytes_per_mbox); - - adjusted_bytes_per_mbox = bytes_per_mbox + config.queue_elements_per_connection * msg_maxsize; - adjusted_bytes_per_mbox = ((adjusted_bytes_per_mbox + CACHELINE_SIZE - 1) / CACHELINE_SIZE) * CACHELINE_SIZE; - - log_debug(nnti_debug_level, "GNI_SmsgBufferSizeNeeded says %d credits needs bytes_per_mbox=%u. Adjusting to %u.", - config.queue_elements_per_connection, bytes_per_mbox, adjusted_bytes_per_mbox); - - c->send_mbox = (nnti_gni_req_queue_mbox_t*)calloc(1,sizeof(nnti_gni_req_queue_mbox_t)); - - c->send_mbox->reg_buf = q->reg_buf; - c->send_mbox->mbox_local_attrs.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT; - c->send_mbox->mbox_local_attrs.buff_size = adjusted_bytes_per_mbox; - c->send_mbox->mbox_local_attrs.mbox_offset = 0; - c->send_mbox->mbox_local_attrs.mbox_maxcredit = config.queue_elements_per_connection; - c->send_mbox->mbox_local_attrs.msg_maxsize = msg_maxsize; - - c->send_mbox->mbox_local_attrs.msg_buffer=(char *)malloc(c->send_mbox->mbox_local_attrs.buff_size); - - rc=GNI_MemRegister( - transport_global_data.nic_hdl, - (uint64_t)c->send_mbox->mbox_local_attrs.msg_buffer, - c->send_mbox->mbox_local_attrs.buff_size, - transport_global_data.req_send_mem_cq_hdl, - GNI_MEM_READWRITE, - -1, - &c->send_mbox->mbox_local_attrs.mem_hndl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "MemRegister(c->send_mbox->mbox_local_attrs.msg_buffer) failed: %d", rc); - } - - return(rc); -} - -static int build_recv_mbox( - nnti_gni_connection_t *c, - uint32_t msg_maxsize) -{ - int rc; - - nnti_gni_request_queue_handle_t *q=&transport_global_data.req_queue; - - const unsigned int CACHELINE_SIZE=64; - - unsigned int bytes_per_mbox=0; - unsigned int adjusted_bytes_per_mbox=0; - - gni_smsg_attr_t smsg_attributes; - - - smsg_attributes.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT; - smsg_attributes.mbox_maxcredit = config.queue_elements_per_connection; - smsg_attributes.msg_maxsize = msg_maxsize; - - rc=GNI_SmsgBufferSizeNeeded(&smsg_attributes, &bytes_per_mbox); - - adjusted_bytes_per_mbox = bytes_per_mbox + config.queue_elements_per_connection * msg_maxsize; - adjusted_bytes_per_mbox = ((adjusted_bytes_per_mbox + CACHELINE_SIZE - 1) / CACHELINE_SIZE) * CACHELINE_SIZE; - - log_debug(nnti_debug_level, "GNI_SmsgBufferSizeNeeded says %d credits needs bytes_per_mbox=%u. Adjusting to %u.", - config.queue_elements_per_connection, bytes_per_mbox, adjusted_bytes_per_mbox); - - c->recv_mbox = (nnti_gni_req_queue_mbox_t*)calloc(1,sizeof(nnti_gni_req_queue_mbox_t)); - - c->recv_mbox->reg_buf = q->reg_buf; - c->recv_mbox->mbox_local_attrs.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT; - c->recv_mbox->mbox_local_attrs.buff_size = adjusted_bytes_per_mbox; - c->recv_mbox->mbox_local_attrs.mbox_offset = 0; - c->recv_mbox->mbox_local_attrs.mbox_maxcredit = config.queue_elements_per_connection; - c->recv_mbox->mbox_local_attrs.msg_maxsize = msg_maxsize; - - c->recv_mbox->mbox_local_attrs.msg_buffer=(char *)malloc(c->recv_mbox->mbox_local_attrs.buff_size); - - rc=GNI_MemRegister( - transport_global_data.nic_hdl, - (uint64_t)c->recv_mbox->mbox_local_attrs.msg_buffer, - c->recv_mbox->mbox_local_attrs.buff_size, - transport_global_data.req_recv_mem_cq_hdl, - GNI_MEM_READWRITE, - -1, - &c->recv_mbox->mbox_local_attrs.mem_hndl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "MemRegister(c->recv_mbox->mbox_local_attrs.msg_buffer) failed: %d", rc); - } - - return(rc); -} - -static int create_loopback( - nnti_gni_connection_t *c) -{ - int rc=0; - - nnti_gni_request_queue_handle_t *q =&transport_global_data.req_queue; - nnti_gni_memory_handle_t *gni_mem_hdl=(nnti_gni_memory_handle_t *)q->reg_buf->transport_private; - - c->peer_name = strdup(transport_global_data.listen_name); - c->peer_addr = (uint32_t)transport_global_data.listen_addr; - c->peer_port = (uint32_t)transport_global_data.listen_port; - c->peer_instance = transport_global_data.instance; - c->peer_alps_info = transport_global_data.alps_info; - c->atomics_addr = (uint64_t)transport_global_data.atomics; - c->atomics_mem_hdl = transport_global_data.atomics_mem_hdl; - - build_send_mbox(c, q->reg_buf->payload_size+gni_mem_hdl->extra); - build_recv_mbox(c, q->reg_buf->payload_size+gni_mem_hdl->extra); - - c->send_mbox->mbox_remote_attrs = c->recv_mbox->mbox_local_attrs; - c->recv_mbox->mbox_remote_attrs = c->send_mbox->mbox_local_attrs; - - client_req_queue_init(c); - server_req_queue_add_client(c); - - rc=GNI_EpCreate (c->nic_hdl, transport_global_data.ep_cq_hdl, &c->ep_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpCreate(c->ep_hdl) failed: %d", rc); - goto out; - } - rc=GNI_EpBind (c->ep_hdl, c->peer_alps_info.local_addr, c->peer_instance); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpBind(c->ep_hdl) failed: %d", rc); - goto out; - } - - log_debug(nnti_debug_level, "new connection ep_hdl(%llu)", c->ep_hdl); - -out: - return(rc); -} - -static int new_client_connection( - nnti_gni_connection_t *c, - int sock) -{ - int rc; - - nnti_gni_request_queue_handle_t *q =&transport_global_data.req_queue; - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - /* - * Values passed through TCP to permit Gemini connection - */ - struct { - char name[NNTI_HOSTNAME_LEN]; - uint32_t addr; - uint32_t port; - NNTI_instance_id instance; - alpsAppGni_t alps_info; - - uint64_t atomics_addr; - gni_mem_handle_t atomics_mem_hdl; - - uint32_t server_msg_maxsize; - - uint32_t client_has_recv_queue; - uint32_t client_msg_maxsize; - } instance_in, instance_out; - struct { - gni_smsg_attr_t mbox_attrs; - } sa_in, sa_out; - - trios_declare_timer(call_time); - - c->connection_type=CLIENT_CONNECTION; - - /* - * Prepare to exchange TCP and ALPS parameters with the server - */ - memset(&instance_out, 0, sizeof(instance_out)); - strcpy(instance_out.name, transport_global_data.listen_name); - instance_out.addr = htonl((uint32_t)transport_global_data.listen_addr); - instance_out.port = htonl((uint32_t)transport_global_data.listen_port); - instance_out.instance = htonl(transport_global_data.instance); - instance_out.alps_info = transport_global_data.alps_info; - - instance_out.atomics_addr =(uint64_t)transport_global_data.atomics; - instance_out.atomics_mem_hdl=transport_global_data.atomics_mem_hdl; - - if ((transport_global_data.req_queue.reg_buf != NULL) && - (transport_global_data.req_queue.req_size > 0) && - (transport_global_data.req_queue.req_count > 0)) { - - gni_mem_hdl=(nnti_gni_memory_handle_t *)q->reg_buf->transport_private; - - instance_out.client_has_recv_queue = 1; - instance_out.client_msg_maxsize = q->reg_buf->payload_size+gni_mem_hdl->extra; - } - - /* - * Exchange TCP and ALPS parameters with the server - */ - trios_start_timer(call_time); - rc = tcp_exchange(sock, 0, &instance_in, &instance_out, sizeof(instance_in)); - trios_stop_timer("tcp_exchange", call_time); - if (rc) - goto out; - - /* - * Record the server's parameters - */ - c->peer_name = strdup(instance_in.name); - c->peer_addr = ntohl(instance_in.addr); - c->peer_port = ntohl(instance_in.port); - c->peer_instance = ntohl(instance_in.instance); - c->peer_alps_info = instance_in.alps_info; - c->atomics_addr = instance_in.atomics_addr; - c->atomics_mem_hdl = instance_in.atomics_mem_hdl; - - - build_send_mbox(c, instance_in.server_msg_maxsize); - - /* - * Write the receive queue attributes to the client - */ - sa_out.mbox_attrs = c->send_mbox->mbox_local_attrs; - - /* - * Read the receive queue attributes from the server - */ - memset(&sa_in, 0, sizeof(sa_in)); - trios_start_timer(call_time); - rc = tcp_exchange(sock, 0, &sa_in, &sa_out, sizeof(sa_in)); - trios_stop_timer("read server queue attrs", call_time); - if (rc == sizeof(sa_in)) { - rc=0; - } - if (rc) - goto out; - - /* - * Record the server's attributes - */ - c->send_mbox->mbox_remote_attrs = sa_in.mbox_attrs; - - /* - * Setup flow control attributes - */ - client_req_queue_init(c); - - - /* - * If the client registered a receive queue (bidirectional connection)... - */ - if (instance_out.client_has_recv_queue == 1) { - - build_recv_mbox(c, instance_out.client_msg_maxsize); - - /* - * Write the receive queue attributes to the client - */ - sa_out.mbox_attrs = c->recv_mbox->mbox_local_attrs; - - trios_start_timer(call_time); - rc = tcp_exchange(sock, 1, &sa_in, &sa_out, sizeof(sa_in)); - trios_stop_timer("write server queue attrs", call_time); - if (rc == sizeof(sa_out)) { - rc=0; - } - if (rc) - goto out; - - c->recv_mbox->mbox_remote_attrs = sa_in.mbox_attrs; - - server_req_queue_add_client(c); - } - - rc=GNI_EpCreate (c->nic_hdl, transport_global_data.ep_cq_hdl, &c->ep_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpCreate(c->ep_hdl) failed: %d", rc); - goto out; - } - rc=GNI_EpBind (c->ep_hdl, c->peer_alps_info.local_addr, c->peer_instance); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpBind(c->ep_hdl) failed: %d", rc); - goto out; - } - - log_debug(nnti_debug_level, "new connection ep_hdl(%llu)", c->ep_hdl); - -out: - return rc; -} - -static int new_server_connection( - nnti_gni_connection_t *c, - int sock) -{ - int rc=0; - - nnti_gni_request_queue_handle_t *q =&transport_global_data.req_queue; - nnti_gni_memory_handle_t *gni_mem_hdl=(nnti_gni_memory_handle_t *)q->reg_buf->transport_private; - - /* - * Values passed through TCP to permit Gemini connection - */ - struct { - char name[NNTI_HOSTNAME_LEN]; - uint32_t addr; - uint32_t port; - NNTI_instance_id instance; - alpsAppGni_t alps_info; - - uint64_t atomics_addr; - gni_mem_handle_t atomics_mem_hdl; - - uint32_t server_msg_maxsize; - - uint32_t client_has_recv_queue; - uint32_t client_msg_maxsize; - } instance_in, instance_out; - struct { - gni_smsg_attr_t mbox_attrs; - } sa_out, sa_in; - - trios_declare_timer(call_time); - - assert(transport_global_data.req_queue.reg_buf); - - c->connection_type=SERVER_CONNECTION; - - /* - * Prepare to exchange TCP and ALPS parameters with the client - */ - memset(&instance_out, 0, sizeof(instance_out)); - strcpy(instance_out.name, transport_global_data.listen_name); - instance_out.addr = htonl((uint32_t)transport_global_data.listen_addr); - instance_out.port = htonl((uint32_t)transport_global_data.listen_port); - instance_out.instance = htonl(transport_global_data.instance); - instance_out.alps_info = transport_global_data.alps_info; - - instance_out.atomics_addr =(uint64_t)transport_global_data.atomics; - instance_out.atomics_mem_hdl=transport_global_data.atomics_mem_hdl; - - instance_out.server_msg_maxsize=q->reg_buf->payload_size+gni_mem_hdl->extra; - - /* - * Exchange TCP and ALPS parameters with the client - */ - trios_start_timer(call_time); - rc = tcp_exchange(sock, 1, &instance_in, &instance_out, sizeof(instance_in)); - trios_stop_timer("tcp_exchange", call_time); - if (rc) - goto out; - - /* - * Record the client's parameters - */ - c->peer_name = strdup(instance_in.name); - c->peer_addr = ntohl(instance_in.addr); - c->peer_port = ntohl(instance_in.port); - c->peer_instance = ntohl(instance_in.instance); - c->peer_alps_info = instance_in.alps_info; - c->peer_ptag = instance_in.alps_info.ptag; - c->peer_cookie = instance_in.alps_info.cookie; - c->atomics_addr = instance_in.atomics_addr; - c->atomics_mem_hdl = instance_in.atomics_mem_hdl; - - c->cdm_hdl = transport_global_data.cdm_hdl; - c->nic_hdl = transport_global_data.nic_hdl; - - build_recv_mbox(c, instance_out.server_msg_maxsize); - - /* - * Write the receive queue attributes to the client - */ - sa_out.mbox_attrs = c->recv_mbox->mbox_local_attrs; - - trios_start_timer(call_time); - rc = tcp_exchange(sock, 1, &sa_in, &sa_out, sizeof(sa_in)); - trios_stop_timer("write server queue attrs", call_time); - if (rc == sizeof(sa_out)) { - rc=0; - } - if (rc) - goto out; - - c->recv_mbox->mbox_remote_attrs = sa_in.mbox_attrs; - - server_req_queue_add_client(c); - - /* - * If the client registered a receive queue (bidirectional connection)... - */ - if (instance_in.client_has_recv_queue == 1) { - - build_send_mbox(c, instance_in.client_msg_maxsize); - - /* - * Write the receive queue attributes to the client - */ - sa_out.mbox_attrs = c->send_mbox->mbox_local_attrs; - - /* - * Read the receive queue attributes from the server - */ - memset(&sa_in, 0, sizeof(sa_in)); - trios_start_timer(call_time); - rc = tcp_exchange(sock, 0, &sa_in, &sa_out, sizeof(sa_in)); - trios_stop_timer("read server queue attrs", call_time); - if (rc == sizeof(sa_in)) { - rc=0; - } - if (rc) - goto out; - - /* - * Record the server's attributes - */ - c->send_mbox->mbox_remote_attrs = sa_in.mbox_attrs; - - /* - * Setup flow control attributes - */ - client_req_queue_init(c); - } - - rc=GNI_EpCreate (c->nic_hdl, transport_global_data.ep_cq_hdl, &c->ep_hdl); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpCreate(c->ep_hdl) failed: %d", rc); - goto out; - } - rc=GNI_EpBind (c->ep_hdl, c->peer_alps_info.local_addr, c->peer_instance); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "EpBind(c->ep_hdl) failed: %d", rc); - goto out; - } - - log_debug(nnti_debug_level, "new connection ep_hdl(%llu)", c->ep_hdl); - -out: - return rc; -} - -/** - * @brief initialize - */ -static NNTI_result_t init_connection( - nnti_gni_connection_t **conn, - const int sock, - const int is_server) -{ - int rc=NNTI_OK; /* return code */ - - trios_declare_timer(call_time); - - log_debug(nnti_debug_level, "initializing gni connection"); - - trios_start_timer(call_time); - if (is_server) { - rc = new_server_connection(*conn, sock); - } else { - rc = new_client_connection(*conn, sock); - } - trios_stop_timer("new connection", call_time); - if (rc) { - close_connection(*conn); - goto out; - } - - print_gni_conn(*conn); - -out: - return((NNTI_result_t)rc); -} - -static int check_for_waiting_connection() -{ - NNTI_result_t rc = NNTI_OK; - - struct sockaddr_in ssin; - socklen_t len; - int s; - nnti_gni_connection_t *conn = NULL; - NNTI_peer_t peer; - - len = sizeof(ssin); - s = accept(transport_global_data.listen_sock, (struct sockaddr *) &ssin, &len); - if (s < 0) { - if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { - log_debug(nnti_debug_level, "no connections waiting to be accepted: %s", strerror(errno)); - rc = NNTI_EWOULDBLOCK; - goto cleanup; - } else { - log_error(nnti_debug_level, "failed to accept tcp socket connection: %s", strerror(errno)); - rc = NNTI_EIO; - goto cleanup; - } - } else { - conn = (nnti_gni_connection_t *)calloc(1, sizeof(nnti_gni_connection_t)); - log_debug(nnti_debug_level, "calloc returned conn=%p.", conn); - if (conn == NULL) { - log_error(nnti_debug_level, "calloc returned NULL. out of memory?: %s", strerror(errno)); - rc=NNTI_ENOMEM; - goto cleanup; - } - nthread_counter_init(&conn->reqs_received); - -// nthread_lock(&nnti_gni_lock); - rc=init_connection(&conn, s, 1); - if (rc!=NNTI_OK) { - goto cleanup; - } - create_peer( - &peer, - conn->peer_name, - conn->peer_addr, - conn->peer_port, - conn->peer_ptag, - conn->peer_cookie, - conn->peer_instance); - - conn->peer=peer; - - del_conn_peer(&peer); - del_conn_instance(conn->peer_instance); - insert_conn_peer(&peer, conn); - insert_conn_instance(conn->peer_instance, conn); - - transition_connection_to_ready(s, 1, conn); -// nthread_unlock(&nnti_gni_lock); - - log_debug(nnti_debug_level, "accepted new connection from %s:%u", conn->peer_name, conn->peer_port); - - if (close(s) < 0) { - log_error(nnti_debug_level, "failed to close new tcp socket"); - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "end of check_listen_socket_for_new_connections", &peer); - } - } - -cleanup: - if (rc != NNTI_OK) { - if (conn!=NULL) free(conn); - } - return rc; -} - -/** - * Check for new connections. The listening socket is left nonblocking - * so this test can be quick; but accept is not really that quick compared - * to polling an Gemini interface, for instance. Returns >0 if an accept worked. - */ -static int check_listen_socket_for_new_connections() -{ - bool done=false; - while(!done) { - if (check_for_waiting_connection() != NNTI_OK) { - done=true; - } - } - - return(NNTI_OK); -} - -static int init_server_listen_socket() -{ - NNTI_result_t rc=NNTI_OK; - trios_declare_timer(call_time); - int flags; - struct hostent *host_entry; - struct sockaddr_in skin; - socklen_t skin_size=sizeof(struct sockaddr_in); - - trios_start_timer(call_time); - transport_global_data.listen_sock = socket(AF_INET, SOCK_STREAM, 0); - trios_stop_timer("socket", call_time); - if (transport_global_data.listen_sock < 0) - log_error(nnti_debug_level, "failed to create tcp socket: %s", strerror(errno)); - - flags = 1; - trios_start_timer(call_time); - if (setsockopt(transport_global_data.listen_sock, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof(flags)) < 0) - log_error(nnti_debug_level, "failed to set tcp socket REUSEADDR flag: %s", strerror(errno)); - trios_stop_timer("setsockopt", call_time); - - flags=fcntl(transport_global_data.listen_sock, F_GETFL, 0); - fcntl(transport_global_data.listen_sock, F_SETFL, flags | O_NONBLOCK); - - log_debug(nnti_debug_level, "listen_name (%s).", transport_global_data.listen_name); - if (transport_global_data.listen_name[0]!='\0') { - log_debug(nnti_debug_level, "using hostname from command-line (%s).", transport_global_data.listen_name); - } else { - trios_start_timer(call_time); - gethostname(transport_global_data.listen_name, NNTI_HOSTNAME_LEN); - trios_stop_timer("gethostname", call_time); - log_debug(nnti_debug_level, "hostname not given on command-line. using gethostname() result (%s).", transport_global_data.listen_name); - } - - /* lookup the host provided on the command line */ - trios_start_timer(call_time); - host_entry = gethostbyname(transport_global_data.listen_name); - trios_stop_timer("gethostbyname", call_time); - if (!host_entry) { - log_warn(nnti_debug_level, "failed to resolve server name (%s): %s", transport_global_data.listen_name, strerror(errno)); - return NNTI_ENOENT; - } - - memset(&skin, 0, sizeof(skin)); - skin.sin_family = AF_INET; - memcpy(&skin.sin_addr, host_entry->h_addr_list[0], (size_t) host_entry->h_length); - /* 0 here means to bind to a random port assigned by the kernel */ - skin.sin_port = 0; - -retry: - trios_start_timer(call_time); - if (bind(transport_global_data.listen_sock, (struct sockaddr *)&skin, skin_size) < 0) { - if (errno == EINTR) { - goto retry; - } else { - log_error(nnti_debug_level, "failed to bind tcp socket: %s", strerror(errno)); - } - } - trios_stop_timer("bind", call_time); - - - /* after the bind, get the "name" for the socket. the "name" contains the port assigned by the kernel. */ - trios_start_timer(call_time); - getsockname(transport_global_data.listen_sock, (struct sockaddr *)&skin, &skin_size); - trios_stop_timer("getsockname", call_time); - transport_global_data.listen_addr = (uint32_t)skin.sin_addr.s_addr; - transport_global_data.listen_port = (uint16_t)skin.sin_port; - log_debug(nnti_debug_level, "listening on ip(%s) addr(%u) port(%u)", - transport_global_data.listen_name, - (unsigned int)ntohl(skin.sin_addr.s_addr), - (unsigned int)ntohs(skin.sin_port)); - trios_start_timer(call_time); - if (listen(transport_global_data.listen_sock, 1024) < 0) - log_error(nnti_debug_level, "failed to listen on tcp socket: %s", strerror(errno)); - trios_stop_timer("listen", call_time); - - return rc; -} - -/* - * At an explicit BYE message, or at finalize time, shut down a connection. - * If descriptors are posted, defer and clean up the connection structures - * later. - */ -static void close_connection(nnti_gni_connection_t *c) -{ - log_level debug_level = nnti_debug_level; // nnti_ee_debug_level; - - if (c==NULL) return; - - log_debug(debug_level, "close_connection: start"); - - print_gni_conn(c); - - c->state=DISCONNECTED; - - nthread_counter_fini(&c->reqs_received); - - if (c->peer_name) { - free(c->peer_name); - c->peer_name = NULL; - } - - if (c->connection_type == CLIENT_CONNECTION) { - client_req_queue_destroy(c); - } - if (c->connection_type == SERVER_CONNECTION) { - server_req_queue_remove_client(c); - } - - log_debug(debug_level, "close_connection: exit"); -} - -static void close_all_conn(void) -{ - log_level debug_level = nnti_debug_level; - - log_debug(debug_level, "enter (%d instance connections, %d peer connections)", - connections_by_instance.size(), connections_by_peer.size()); - - if (nthread_lock(&nnti_conn_instance_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - conn_by_inst_iter_t inst_iter = connections_by_instance.begin(); - while (inst_iter != connections_by_instance.end()) { - log_debug(debug_level, "close connection (instance=%llu)", inst_iter->first); - close_connection(inst_iter->second); - - connections_by_instance.erase(inst_iter++); - } - nthread_unlock(&nnti_conn_instance_lock); - - if (nthread_lock(&nnti_conn_peer_lock)) log_warn(nnti_debug_level, "failed to get thread lock"); - conn_by_peer_iter_t peer_iter = connections_by_peer.begin(); - while (peer_iter != connections_by_peer.end()) { - log_debug(debug_level, "close connection (peer.addr=%llu)", peer_iter->first.addr); -// close_connection(peer_iter->second); - - connections_by_peer.erase(peer_iter++); - } - nthread_unlock(&nnti_conn_peer_lock); - - log_debug(debug_level, "exit (%d instance connections, %d peer connections)", - connections_by_instance.size(), connections_by_peer.size()); - - return; -} - -static uint32_t get_cpunum(void) -{ - int i, j; - uint32_t cpu_num; - - cpu_set_t coremask; - - (void)sched_getaffinity(0, sizeof(coremask), &coremask); - - for (i = 0; i < CPU_SETSIZE; i++) { - if (CPU_ISSET(i, &coremask)) { - int run = 0; - for (j = i + 1; j < CPU_SETSIZE; j++) { - if (CPU_ISSET(j, &coremask)) run++; - else break; - } - if (!run) { - cpu_num=i; - } else { - fprintf(stdout, "This thread is bound to multiple CPUs(%d). Using lowest numbered CPU(%d).", run+1, cpu_num); - cpu_num=i; - } - } - } - return(cpu_num); -} - -static void get_alps_info( - alpsAppGni_t *alps_info) -{ - int alps_rc=0; - int req_rc=0; - size_t rep_size=0; - - uint64_t apid=0; - alpsAppLLIGni_t *alps_info_list; - char buf[1024]; - - char *ptag_env_str=NULL; - char *cookie_env_str=NULL; - - alps_info_list=(alpsAppLLIGni_t *)&buf[0]; - - alps_app_lli_lock(); - - log_debug(nnti_debug_level, "sending ALPS request"); - alps_rc = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_GNI, NULL, 0); - if (alps_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_put_request failed: %d", alps_rc); - log_debug(nnti_debug_level, "waiting for ALPS reply"); - alps_rc = alps_app_lli_get_response(&req_rc, &rep_size); - if (alps_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_get_response failed: alps_rc=%d", alps_rc); - if (req_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_get_response failed: req_rc=%d", req_rc); - if (rep_size != 0) { - log_debug(nnti_debug_level, "waiting for ALPS reply bytes (%d) ; sizeof(alps_info)==%d ; sizeof(alps_info_list)==%d", rep_size, sizeof(alps_info), sizeof(alps_info_list)); - alps_rc = alps_app_lli_get_response_bytes(alps_info_list, rep_size); - if (alps_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_get_response_bytes failed: %d", alps_rc); - } - - log_debug(nnti_debug_level, "sending ALPS request"); - alps_rc = alps_app_lli_put_request(ALPS_APP_LLI_ALPS_REQ_APID, NULL, 0); - if (alps_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_put_request failed: %d", alps_rc); - log_debug(nnti_debug_level, "waiting for ALPS reply"); - alps_rc = alps_app_lli_get_response(&req_rc, &rep_size); - if (alps_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_get_response failed: alps_rc=%d", alps_rc); - if (req_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_get_response failed: req_rc=%d", req_rc); - if (rep_size != 0) { - log_debug(nnti_debug_level, "waiting for ALPS reply bytes (%d) ; sizeof(apid)==%d", rep_size, sizeof(apid)); - alps_rc = alps_app_lli_get_response_bytes(&apid, rep_size); - if (alps_rc != 0) log_debug(nnti_debug_level, "alps_app_lli_get_response_bytes failed: %d", alps_rc); - } - - alps_app_lli_unlock(); - - memcpy(alps_info, (alpsAppGni_t*)alps_info_list->u.buf, sizeof(alpsAppGni_t)); - transport_global_data.apid=apid; - - log_debug(nnti_debug_level, "apid =%llu", (unsigned long long)apid); - log_debug(nnti_debug_level, "alps_info->device_id =%llu", (unsigned long long)alps_info->device_id); - log_debug(nnti_debug_level, "alps_info->local_addr=%lld", (long long)alps_info->local_addr); - log_debug(nnti_debug_level, "alps_info->cookie =%llu", (unsigned long long)alps_info->cookie); - log_debug(nnti_debug_level, "alps_info->ptag =%llu", (unsigned long long)alps_info->ptag); - - log_debug(nnti_debug_level, "ALPS response - apid(%llu) alps_info->device_id(%llu) alps_info->local_addr(%llu) " - "alps_info->cookie(%llu) alps_info->ptag(%llu)", - (unsigned long long)apid, - (unsigned long long)alps_info->device_id, - (long long)alps_info->local_addr, - (unsigned long long)alps_info->cookie, - (unsigned long long)alps_info->ptag); - - if ((ptag_env_str=getenv("TRIOS_NNTI_GNI_PTAG")) != NULL) { - uint32_t ptag_env=strtoul(ptag_env_str, NULL, 0); - log_debug(nnti_debug_level, "replacing ALPS ptag (%lu) with user supplied ptag (%lu)", alps_info->ptag, ptag_env); - alps_info->ptag=ptag_env; - } - if ((cookie_env_str=getenv("TRIOS_NNTI_GNI_COOKIE")) != NULL) { - uint32_t cookie_env=strtoul(cookie_env_str, NULL, 0); - log_debug(nnti_debug_level, "replacing ALPS cookie (%lu) with user supplied cookie (%lu)", alps_info->cookie, cookie_env); - alps_info->cookie=cookie_env; - } - if ((ptag_env_str!=NULL) && (cookie_env_str==NULL)) { - log_warn(nnti_debug_level, "user supplied ptag (%s) found without a user supplied cookie. this probably won't work.", ptag_env_str); - } - if ((ptag_env_str==NULL) && (cookie_env_str!=NULL)) { - log_warn(nnti_debug_level, "user supplied cookie (%s) found without a user supplied ptag. this probably won't work.", cookie_env_str); - } - - return; -} - -static void print_wc(const nnti_gni_work_completion_t *wc) -{ - if (!logging_debug(nnti_debug_level)) { - return; - } - - log_debug(nnti_debug_level, "wc=%p, wc.op=%d, wc.inst_id=%llu, wc.byte_len=%llu, wc.byte_offset=%llu, wc.src_index=%llu, wc.src_offset=%llu, wc.dest_index=%llu, wc.dest_offset=%llu", - wc, - wc->op, - wc->inst_id, - wc->byte_len, - wc->byte_offset, - wc->src_index, - wc->src_offset, - wc->dest_index, - wc->dest_offset); -} - -static void print_cq_event( - const gni_cq_entry_t *event, - const bool force) -{ - if (force) { - log_debug(LOG_ALL, "event=%p, event.data=%llu, event.source=%llu, event.status=%llu, " - "event.info=%llu, event.overrun=%llu, event.inst_id=%llu, event.tid=%llu, event.msg_id=%llu, event.type=%llu", - event, - (uint64_t)gni_cq_get_data(*event), - (uint64_t)gni_cq_get_source(*event), - (uint64_t)gni_cq_get_status(*event), - (uint64_t)gni_cq_get_info(*event), - (uint64_t)gni_cq_overrun(*event), - (uint64_t)gni_cq_get_inst_id(*event), - (uint64_t)gni_cq_get_tid(*event), - (uint64_t)gni_cq_get_msg_id(*event), - (uint64_t)gni_cq_get_type(*event)); - } else if (gni_cq_get_status(*event) != 0) { - log_error(nnti_debug_level, "event=%p, event.data=%llu, event.source=%llu, event.status=%llu, " - "event.info=%llu, event.overrun=%llu, event.inst_id=%llu, event.tid=%llu, event.msg_id=%llu, event.type=%llu", - event, - (uint64_t)gni_cq_get_data(*event), - (uint64_t)gni_cq_get_source(*event), - (uint64_t)gni_cq_get_status(*event), - (uint64_t)gni_cq_get_info(*event), - (uint64_t)gni_cq_overrun(*event), - (uint64_t)gni_cq_get_inst_id(*event), - (uint64_t)gni_cq_get_tid(*event), - (uint64_t)gni_cq_get_msg_id(*event), - (uint64_t)gni_cq_get_type(*event)); - } else { - log_debug(nnti_debug_level, "event=%p, event.data=%llu, event.source=%llu, event.status=%llu, " - "event.info=%llu, event.overrun=%llu, event.inst_id=%llu, event.tid=%llu, event.msg_id=%llu, event.type=%llu", - event, - (uint64_t)gni_cq_get_data(*event), - (uint64_t)gni_cq_get_source(*event), - (uint64_t)gni_cq_get_status(*event), - (uint64_t)gni_cq_get_info(*event), - (uint64_t)gni_cq_overrun(*event), - (uint64_t)gni_cq_get_inst_id(*event), - (uint64_t)gni_cq_get_tid(*event), - (uint64_t)gni_cq_get_msg_id(*event), - (uint64_t)gni_cq_get_type(*event)); - } -} - -static void print_post_desc( - const gni_post_descriptor_t *post_desc_ptr) -{ - if (!logging_debug(nnti_debug_level)) { - return; - } - - if (post_desc_ptr != NULL) { - log_debug(nnti_debug_level, "post_desc_ptr ==%p", (uint64_t)post_desc_ptr); - log_debug(nnti_debug_level, "post_desc_ptr->next_descr ==%p", (uint64_t)post_desc_ptr->next_descr); - log_debug(nnti_debug_level, "post_desc_ptr->prev_descr ==%p", (uint64_t)post_desc_ptr->prev_descr); - - log_debug(nnti_debug_level, "post_desc_ptr->post_id ==%llu", (uint64_t)post_desc_ptr->post_id); - log_debug(nnti_debug_level, "post_desc_ptr->status ==%llu", (uint64_t)post_desc_ptr->status); - log_debug(nnti_debug_level, "post_desc_ptr->cq_mode_complete==%llu", (uint64_t)post_desc_ptr->cq_mode_complete); - - log_debug(nnti_debug_level, "post_desc_ptr->type ==%llu", (uint64_t)post_desc_ptr->type); - log_debug(nnti_debug_level, "post_desc_ptr->cq_mode ==%llu", (uint64_t)post_desc_ptr->cq_mode); - log_debug(nnti_debug_level, "post_desc_ptr->dlvr_mode ==%llu", (uint64_t)post_desc_ptr->dlvr_mode); - log_debug(nnti_debug_level, "post_desc_ptr->local_addr ==%llu", (uint64_t)post_desc_ptr->local_addr); - log_debug(nnti_debug_level, "post_desc_ptr->remote_addr ==%llu", (uint64_t)post_desc_ptr->remote_addr); - log_debug(nnti_debug_level, "post_desc_ptr->length ==%llu", (uint64_t)post_desc_ptr->length); - log_debug(nnti_debug_level, "post_desc_ptr->rdma_mode ==%llu", (uint64_t)post_desc_ptr->rdma_mode); - log_debug(nnti_debug_level, "post_desc_ptr->src_cq_hndl ==%llu", (uint64_t)post_desc_ptr->src_cq_hndl); - log_debug(nnti_debug_level, "post_desc_ptr->sync_flag_value ==%llu", (uint64_t)post_desc_ptr->sync_flag_value); - log_debug(nnti_debug_level, "post_desc_ptr->sync_flag_addr ==%llu", (uint64_t)post_desc_ptr->sync_flag_addr); - log_debug(nnti_debug_level, "post_desc_ptr->amo_cmd ==%llu", (uint64_t)post_desc_ptr->amo_cmd); - log_debug(nnti_debug_level, "post_desc_ptr->first_operand ==%llu", (uint64_t)post_desc_ptr->first_operand); - log_debug(nnti_debug_level, "post_desc_ptr->second_operand ==%llu", (uint64_t)post_desc_ptr->second_operand); - log_debug(nnti_debug_level, "post_desc_ptr->cqwrite_value ==%llu", (uint64_t)post_desc_ptr->cqwrite_value); - } else { - log_debug(nnti_debug_level, "post_desc_ptr == NULL"); - } -} - -static void print_gni_conn(nnti_gni_connection_t *c) -{ - log_level debug_level=nnti_debug_level; - - if (!logging_debug(debug_level)) { - return; - } - - log_debug(debug_level, "c->peer_name =%s", c->peer_name); - log_debug(debug_level, "c->peer_addr =%u", c->peer_addr); - log_debug(debug_level, "c->peer_port =%u", (uint32_t)c->peer_port); - log_debug(debug_level, "c->peer_instance =%llu", (uint64_t)c->peer_instance); - - log_debug(debug_level, "c->state =%d", c->state); -} - -//static void print_put_buf(void *buf, uint32_t size) -//{ -// struct data_t { -// uint32_t int_val; -// float float_val; -// double double_val; -// }; -// -// struct data_array_t { -// u_int data_array_t_len; -// struct data_t *data_array_t_val; -// }; -// -//// struct data_array_t *da=(struct data_array_t *)buf; -// const struct data_t *array = (struct data_t *)buf; -// const int len = size/sizeof(struct data_t); -// int idx=0; -// -// for (idx=0;idx ", (uint64_t)buf, l); - fprintf(f, "%02hhX", ((char *)buf)[l]); - } - fprintf(f, "\n"); - } - } -} - -static uint16_t get_dlvr_mode_from_env() -{ - char *mode=getenv("NSSI_GNI_DELIVERY_MODE"); - - log_debug(nnti_debug_level, "NSSI_GNI_DELIVERY_MODE=%s", mode); - if ((mode==NULL) || !strcmp(mode, "GNI_DLVMODE_PERFORMANCE")) { - log_debug(nnti_debug_level, "setting delivery mode to GNI_DLVMODE_PERFORMANCE"); - return GNI_DLVMODE_PERFORMANCE; - } else if (!strcmp(mode, "GNI_DLVMODE_IN_ORDER")) { - log_debug(nnti_debug_level, "setting delivery mode to GNI_DLVMODE_IN_ORDER"); - return GNI_DLVMODE_IN_ORDER; - } else if (!strcmp(mode, "GNI_DLVMODE_NO_ADAPT")) { - log_debug(nnti_debug_level, "setting delivery mode to GNI_DLVMODE_NO_ADAPT"); - return GNI_DLVMODE_NO_ADAPT; - } else if (!strcmp(mode, "GNI_DLVMODE_NO_HASH")) { - log_debug(nnti_debug_level, "setting delivery mode to GNI_DLVMODE_NO_HASH"); - return GNI_DLVMODE_NO_HASH; - } else if (!strcmp(mode, "GNI_DLVMODE_NO_RADAPT")) { - log_debug(nnti_debug_level, "setting delivery mode to GNI_DLVMODE_NO_RADAPT"); - return GNI_DLVMODE_NO_RADAPT; - } else { - log_debug(nnti_debug_level, "defaulting delivery mode to GNI_DLVMODE_PERFORMANCE"); - return GNI_DLVMODE_PERFORMANCE; - } -} -static void set_dlvr_mode( - gni_post_descriptor_t *pd) -{ - pd->dlvr_mode=transport_global_data.delivery_mode; -} - -static void set_rdma_mode( - gni_post_descriptor_t *pd) -{ - if (config.use_rdma_fence==true) { - pd->rdma_mode=GNI_RDMAMODE_FENCE; - } -} - -static void set_post_desc(gni_post_descriptor_t *pd, uint8_t op, uint32_t buf_length) -{ - if (op == GNI_OP_GET_INITIATOR) { - // set type - switch (config.rdma_mode) { - case RDMA_FMA: - pd->type=GNI_POST_FMA_GET; - break; - case RDMA_CROSSOVER: - if (buf_length > config.fma_bte_crossover_size) { - pd->type=GNI_POST_RDMA_GET; - } else { - pd->type=GNI_POST_FMA_GET; - } - break; - case RDMA_BTE: - case RDMA_MIXED: - default: - pd->type=GNI_POST_RDMA_GET; - break; - } - // set cq_mode - if (config.use_rdma_events==true) { - pd->cq_mode=GNI_CQMODE_GLOBAL_EVENT|GNI_CQMODE_REMOTE_EVENT; - } else { - pd->cq_mode=GNI_CQMODE_GLOBAL_EVENT; - } - - } else if (op == GNI_OP_PUT_INITIATOR) { - // set type - switch (config.rdma_mode) { - case RDMA_FMA: - pd->type=GNI_POST_FMA_PUT; - break; - case RDMA_CROSSOVER: - if (buf_length > config.fma_bte_crossover_size) { - pd->type=GNI_POST_RDMA_PUT; - } else { - pd->type=GNI_POST_FMA_PUT; - } - break; - case RDMA_BTE: - case RDMA_MIXED: - default: - pd->type=GNI_POST_RDMA_PUT; - break; - } - // set cq_mode - if (config.use_rdma_events==true) { - pd->cq_mode=GNI_CQMODE_GLOBAL_EVENT|GNI_CQMODE_REMOTE_EVENT; - } else { - pd->cq_mode=GNI_CQMODE_GLOBAL_EVENT; - } - - } else if (op == GNI_OP_SEND_BUFFER) { - // set type - switch (config.rdma_mode) { - case RDMA_CROSSOVER: - if (buf_length > config.fma_bte_crossover_size) { - pd->type=GNI_POST_RDMA_PUT; - } else { - pd->type=GNI_POST_FMA_PUT; - } - break; - case RDMA_BTE: - pd->type=GNI_POST_RDMA_PUT; - break; - case RDMA_FMA: - case RDMA_MIXED: - default: - pd->type=GNI_POST_FMA_PUT; - break; - } - // set cq_mode - pd->cq_mode=GNI_CQMODE_GLOBAL_EVENT|GNI_CQMODE_REMOTE_EVENT; - - } else if (op == GNI_OP_SEND_REQUEST) { - // set type - switch (config.rdma_mode) { - case RDMA_CROSSOVER: - if (buf_length > config.fma_bte_crossover_size) { - pd->type=GNI_POST_RDMA_PUT; - } else { - pd->type=GNI_POST_FMA_PUT; - } - break; - case RDMA_BTE: - pd->type=GNI_POST_RDMA_PUT; - break; - case RDMA_FMA: - case RDMA_MIXED: - default: - pd->type=GNI_POST_FMA_PUT; - break; - } - // set cq_mode - pd->cq_mode=GNI_CQMODE_GLOBAL_EVENT|GNI_CQMODE_REMOTE_EVENT; - } else { - log_error(nnti_debug_level, "unknown op(%u).", op); - } - - set_dlvr_mode(pd); - set_rdma_mode(pd); -} - -static int post_wait( - gni_cq_handle_t cq_hdl, - int timeout) -{ - int rc=0; - gni_post_descriptor_t *post_desc_ptr=NULL; - gni_cq_entry_t ev_data; - - log_debug(nnti_ee_debug_level, "enter"); - -retry: - memset(&ev_data, 0, sizeof(ev_data)); - log_debug(nnti_ee_debug_level, "calling CqWaitEvent"); - nthread_lock(&nnti_gni_lock); - rc=GNI_CqWaitEvent (cq_hdl, timeout, &ev_data); - nthread_unlock(&nnti_gni_lock); - if (rc==GNI_RC_SUCCESS) { - print_cq_event(&ev_data, false); - log_debug(nnti_debug_level, "calling GetComplete"); - nthread_lock(&nnti_gni_lock); - rc=GNI_GetCompleted (cq_hdl, ev_data, &post_desc_ptr); - nthread_unlock(&nnti_gni_lock); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "GetCompleted failed: %d", rc); - print_post_desc(post_desc_ptr); - } else if (rc==GNI_RC_TIMEOUT) { - log_debug(nnti_debug_level, "CqWaitEvent timed out: %d", rc); - goto retry; - } else { - log_error(nnti_debug_level, "CqWaitEvent failed: %d", rc); - } - - log_debug(nnti_ee_debug_level, "exit"); - - return(rc); -} - -static int send_back_credits( - nnti_gni_connection_t *conn, - uint64_t credits_to_send) -{ - /* This is currently a noop. The uGNI docs say that codes that - * only send in one direction may run out of credits because - * credits won't be returned without an explicit send. If we - * find ourselves in that situation we will send some messages - * from here. */ - - return (GNI_RC_SUCCESS); -} - -static int execute_send_wr( - nnti_gni_work_request_t *gni_wr) -{ - int rc=GNI_RC_NOT_DONE; - int credits=0; - - trios_declare_timer(call_time); - - log_debug(nnti_debug_level, "enter (wr=%p gni_wr=%p)", gni_wr->nnti_wr, gni_wr); - - nnti_gni_memory_handle_t *gni_mem_hdl=(nnti_gni_memory_handle_t *)gni_wr->reg_buf->transport_private; - assert(gni_mem_hdl); - - nnti_gni_connection_t *conn=get_conn_instance(gni_wr->peer_instance); - assert(conn); - - trios_start_timer(call_time); - nthread_lock(&nnti_gni_lock); - rc = GNI_SmsgSendWTag( - conn->send_mbox->ep_hdl, - (void *)gni_wr->reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf, - gni_wr->reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size + gni_mem_hdl->extra, - NULL, - 0, - hash6432shift((uint64_t)&gni_wr->sge_list[0]), - NNTI_SMSG_TAG_REQUEST); - nthread_unlock(&nnti_gni_lock); - trios_stop_timer("SmsgSend", call_time); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - -static int insert_resend( - nnti_gni_work_request_t *gni_wr) -{ - wr_queue_t *peer_resend_queue; - - nthread_lock(&nnti_resend_lock); - wr_queue_map_iter_t target=wr_resend_map.find(gni_wr->peer_instance); - if (target != wr_resend_map.end()) { - peer_resend_queue = target->second; - } else { - peer_resend_queue = new wr_queue_t; - wr_resend_map[gni_wr->peer_instance]=peer_resend_queue; - } - - peer_resend_queue->push_back(gni_wr); - nthread_unlock(&nnti_resend_lock); - - return 0; -} - -static int resend_all(void) -{ - int rc=0; - - wr_queue_map_iter_t map_iter; - wr_queue_iter_t queue_iter; - - wr_queue_t *peer_resend_queue; - - nthread_lock(&nnti_resend_lock); - log_debug(nnti_debug_level, "wr_resend_map.size()=%d", wr_resend_map.size()); - - map_iter=wr_resend_map.begin(); - while (map_iter != wr_resend_map.end()) { - peer_resend_queue=map_iter->second; - - log_debug(nnti_debug_level, "peer_resend_queue.size()=%d", peer_resend_queue->size()); - - queue_iter=peer_resend_queue->begin(); - while (queue_iter != peer_resend_queue->end()) { - nnti_gni_work_request_t *gni_wr=*queue_iter; - - rc=execute_send_wr(gni_wr); - if (rc == GNI_RC_SUCCESS) { - log_debug(nnti_debug_level, "execute_send_wr(conn->send_mbox) success: %d", rc); - - queue_iter = peer_resend_queue->erase(queue_iter); - - } else if (rc == GNI_RC_NOT_DONE) { - log_warn(nnti_debug_level, "execute_send_wr(conn->send_mbox) dropped again: %d", rc); - - break; - - } else { - log_warn(nnti_debug_level, "execute_send_wr(conn->send_mbox) failed: %d", rc); - gni_wr->sge_list[0].state=NNTI_GNI_SGE_STATE_ERROR; - gni_wr->state =NNTI_GNI_WR_STATE_ERROR; - gni_wr->nnti_wr->result =NNTI_EIO; - - break; - - } - } - - if (peer_resend_queue->empty()) { - wr_resend_map.erase(map_iter++); - } else { - ++map_iter; - } - } - nthread_unlock(&nnti_resend_lock); - - return rc; -} - -static int request_send( - const NNTI_peer_t *peer_hdl, - nnti_gni_connection_t *conn, - const NNTI_buffer_t *reg_buf, - int req_num, - nnti_gni_work_request_t *gni_wr) -{ - int rc=0; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - log_debug(nnti_ee_debug_level, "enter"); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)reg_buf->transport_private; - assert(gni_mem_hdl); - - gni_wr->reg_buf =reg_buf; - gni_wr->wc =GNI_WC_ADDRESS(reg_buf, 0); - gni_wr->wc->ack_received=0; - gni_wr->wc->inst_id =transport_global_data.instance; - gni_wr->wc->op =GNI_OP_SEND_REQUEST; - gni_wr->wc->byte_len =reg_buf->payload_size; - gni_wr->wc->byte_offset =0; - gni_wr->wc->src_offset =0; - gni_wr->wc->dest_offset =0; - - gni_wr->last_op =GNI_OP_SEND_REQUEST; - gni_wr->state=NNTI_GNI_WR_STATE_STARTED; - - gni_wr->sge_list =&gni_wr->sge; - gni_wr->sge_count=1; - - gni_wr->sge_list[0].gni_wr=gni_wr; - gni_wr->sge_list[0].state=NNTI_GNI_SGE_STATE_STARTED; - - gni_wr->peer_instance=peer_hdl->peer.NNTI_remote_process_t_u.gni.inst_id; - log_debug(nnti_event_debug_level, "gni_wr->peer_instance==%lu", gni_wr->peer_instance); - - nthread_lock(&gni_mem_hdl->wr_queue_lock); - gni_mem_hdl->wr_queue->push_back(gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - insert_sge_sgehash(&gni_wr->sge_list[0]); - - rc=execute_send_wr(gni_wr); - if (rc == GNI_RC_SUCCESS) { - log_debug(nnti_debug_level, "SmsgSend(conn->send_mbox->ep_hdl) success: %d", rc); - } else if (rc == GNI_RC_NOT_DONE) { - log_warn(nnti_debug_level, "SmsgSend(conn->send_mbox->ep_hdl) dropped: %d", rc); - insert_resend(gni_wr); - } else { - log_warn(nnti_debug_level, "SmsgSend(conn->send_mbox->ep_hdl) failed: %d", rc); - gni_wr->sge_list[0].state=NNTI_GNI_SGE_STATE_ERROR; - gni_wr->state =NNTI_GNI_WR_STATE_ERROR; - gni_wr->nnti_wr->result =NNTI_EIO; - } - - log_debug(nnti_ee_debug_level, "exit"); - - return(rc); -} - -static int send_buffer( - const NNTI_peer_t *peer_hdl, - nnti_gni_connection_t *conn, - const NNTI_buffer_t *src_hdl, - const NNTI_buffer_t *dest_hdl, - nnti_gni_work_request_t *gni_wr) -{ - int rc=0; - - gni_ep_handle_t ep_hdl; - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - bool use_fma=true; - - trios_declare_timer(call_time); - - gni_mem_hdl=(nnti_gni_memory_handle_t *)src_hdl->transport_private; - assert(gni_mem_hdl); - - log_debug(nnti_ee_debug_level, "enter"); - - gni_wr->last_op=GNI_OP_SEND_BUFFER; - gni_wr->state=NNTI_GNI_WR_STATE_STARTED; - - gni_wr->sge_list =&gni_wr->sge; - gni_wr->sge_count=1; - - gni_wr->sge_list[0].gni_wr=gni_wr; - gni_wr->sge_list[0].state=NNTI_GNI_SGE_STATE_STARTED; - - memset(&gni_wr->sge_list[0].post_desc, 0, sizeof(gni_post_descriptor_t)); - set_post_desc(&gni_wr->sge_list[0].post_desc, GNI_OP_SEND_BUFFER, src_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size); - - gni_wr->sge_list[0].post_desc.src_cq_hndl=transport_global_data.ep_cq_hdl; - - gni_wr->sge_list[0].post_desc.local_addr =src_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf; - gni_wr->sge_list[0].post_desc.local_mem_hndl.qword1 =src_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword1; - gni_wr->sge_list[0].post_desc.local_mem_hndl.qword2 =src_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword2; - gni_wr->sge_list[0].post_desc.remote_addr =dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf; - gni_wr->sge_list[0].post_desc.remote_mem_hndl.qword1=dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword1; - gni_wr->sge_list[0].post_desc.remote_mem_hndl.qword2=dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword2; - gni_wr->sge_list[0].post_desc.length =src_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size + gni_mem_hdl->extra; - - gni_wr->sge_list[0].post_desc.dlvr_mode=GNI_DLVMODE_IN_ORDER; - - print_raw_buf((void *)gni_wr->sge_list[0].post_desc.local_addr, gni_wr->sge_list[0].post_desc.length); - - insert_sge_sgehash(&gni_wr->sge_list[0]); - - ep_hdl=conn->ep_hdl; - - if (config.rdma_mode==RDMA_CROSSOVER) { - if (gni_wr->sge_list[0].post_desc.length > config.fma_bte_crossover_size) { - use_fma=false; - } else { - use_fma=true; - } - } else if (config.rdma_mode==RDMA_BTE) { - use_fma=false; - } else if ((config.rdma_mode==RDMA_FMA) || (config.rdma_mode==RDMA_MIXED)) { - use_fma=true; - } - -nthread_lock(&nnti_gni_lock); - GNI_EpSetEventData( - ep_hdl, - hash6432shift((uint64_t)&gni_wr->sge_list[0]), - hash6432shift((uint64_t)dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf)); - - if (use_fma) { - log_debug(nnti_debug_level, "calling PostFma(send buffer ep_hdl(%llu), ep_cq_hdl(%llu), local_addr=%llu, remote_addr=%llu)", - ep_hdl, transport_global_data.ep_cq_hdl, gni_wr->sge_list[0].post_desc.local_addr, gni_wr->sge_list[0].post_desc.remote_addr); - trios_start_timer(call_time); - rc=GNI_PostFma(ep_hdl, &gni_wr->sge_list[0].post_desc); - trios_stop_timer("PostFma req", call_time); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "PostFma(fma put) failed: %d", rc); - } else { - log_debug(nnti_debug_level, "calling PostRdma(send buffer ep_hdl(%llu), ep_cq_hdl(%llu), local_addr=%llu, remote_addr=%llu)", - ep_hdl, transport_global_data.ep_cq_hdl, gni_wr->sge_list[0].post_desc.local_addr, gni_wr->sge_list[0].post_desc.remote_addr); - trios_start_timer(call_time); - rc=GNI_PostRdma(ep_hdl, &gni_wr->sge_list[0].post_desc); - trios_stop_timer("PostRdma req", call_time); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "PostRdma(rdma put) failed: %d", rc); - } -nthread_unlock(&nnti_gni_lock); - - log_debug(nnti_ee_debug_level, "exit"); - - return(0); -} - -static int buffer_send( - const NNTI_peer_t *peer_hdl, - nnti_gni_connection_t *conn, - const NNTI_buffer_t *src_hdl, - const NNTI_buffer_t *dest_hdl, - nnti_gni_work_request_t *gni_wr) -{ - int rc=0; - trios_declare_timer(call_time); - - nnti_gni_memory_handle_t *gni_mem_hdl=NULL; - - - gni_mem_hdl=(nnti_gni_memory_handle_t *)src_hdl->transport_private; - assert(gni_mem_hdl); - - log_debug(nnti_ee_debug_level, "enter"); - - gni_wr->reg_buf =src_hdl; - gni_wr->wc =GNI_WC_ADDRESS(src_hdl, 0); - gni_wr->wc->ack_received=0; - gni_wr->wc->inst_id =transport_global_data.instance; - gni_wr->wc->op =GNI_OP_SEND_BUFFER; - gni_wr->wc->byte_len =src_hdl->payload_size; - gni_wr->wc->byte_offset =0; - gni_wr->wc->src_offset =0; - gni_wr->wc->dest_offset =0; - - - nthread_lock(&gni_mem_hdl->wr_queue_lock); - gni_mem_hdl->wr_queue->push_back(gni_wr); - nthread_unlock(&gni_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "calling send_buffer()"); - trios_start_timer(call_time); - rc=send_buffer(peer_hdl, conn, src_hdl, dest_hdl, gni_wr); - trios_stop_timer("send_buffer", call_time); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "send_buffer() failed: %d", rc); - - gni_wr->peer_instance=dest_hdl->buffer_owner.peer.NNTI_remote_process_t_u.gni.inst_id; - log_debug(nnti_event_debug_level, "gni_wr->last_peer==%lu", gni_wr->peer_instance); - - log_debug(nnti_ee_debug_level, "exit"); - - return(0); -} - -static void client_req_queue_init( - nnti_gni_connection_t *c) -{ - int rc; - - log_debug(nnti_debug_level, "enter"); - - rc=GNI_EpCreate ( - c->nic_hdl, - transport_global_data.req_send_ep_cq_hdl, - &c->send_mbox->ep_hdl); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "EpCreate(c->send_mbox->ep_hdl) failed: %d", rc); - rc=GNI_EpBind ( - c->send_mbox->ep_hdl, - c->peer_alps_info.local_addr, - c->peer_instance); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "EpBind(c->send_mbox->ep_hdl) failed: %d", rc); - - rc=GNI_SmsgInit( - c->send_mbox->ep_hdl, - &c->send_mbox->mbox_local_attrs, - &c->send_mbox->mbox_remote_attrs); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "SmsgInit(c->send_mbox->ep_hdl) failed: %d", rc); - - log_debug(nnti_debug_level, "new connection send_mbox->ep_hdl(%llu)", c->send_mbox->ep_hdl); - - log_debug(nnti_debug_level, "exit"); -} - -static void client_req_queue_destroy( - nnti_gni_connection_t *c) -{ - int rc; - log_level debug_level = nnti_debug_level; // nnti_debug_level; - - log_debug(debug_level, "enter"); - - if (c->send_mbox) { - rc=GNI_EpUnbind (c->send_mbox->ep_hdl); - if (rc!=GNI_RC_SUCCESS) log_error(debug_level, "EpUnbind() failed: %d", rc); - rc=GNI_EpDestroy (c->send_mbox->ep_hdl); - if (rc!=GNI_RC_SUCCESS) log_error(debug_level, "EpDestroy() failed: %d", rc); - - rc=GNI_MemDeregister (c->nic_hdl, &c->send_mbox->mbox_local_attrs.mem_hndl); - if (rc!=GNI_RC_SUCCESS) log_error(debug_level, "MemDeregister(1) failed: %d", rc); - - free(c->send_mbox->mbox_local_attrs.msg_buffer); - free(c->send_mbox); - } - - log_debug(debug_level, "exit"); -} - -static void server_req_queue_init( - nnti_gni_request_queue_handle_t *q, - char *buffer, - uint64_t req_size, - uint64_t extra, - uint64_t req_count) -{ - log_debug(nnti_debug_level, "enter"); - - q->req_buffer =buffer; - q->req_size =req_size; - q->req_count =req_count; - q->req_buffer_size=req_count*(req_size+extra); - - nthread_counter_init(&q->req_index); - - q->head=0; - q->tail=0; - - log_debug(nnti_debug_level, "exit"); -} - -static void server_req_queue_add_client( - nnti_gni_connection_t *c) -{ - int rc; - - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "c->peer_alps_info.local_addr(%lld) c->peer_instance(%llu) transport_global_data.ep_cq_hdl(%llu) transport_global_data.mem_cq_hdl(%llu)", - (long long)c->peer_alps_info.local_addr, - (unsigned long long)c->peer_instance, - (unsigned long long)transport_global_data.ep_cq_hdl, - (unsigned long long)transport_global_data.mem_cq_hdl); - - rc=GNI_EpCreate ( - c->nic_hdl, - transport_global_data.req_recv_ep_cq_hdl, - &c->recv_mbox->ep_hdl); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "EpCreate(c->recv_mbox->ep_hdl) failed: %d", rc); - rc=GNI_EpBind ( - c->recv_mbox->ep_hdl, - c->peer_alps_info.local_addr, - c->peer_instance); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "EpBind(c->recv_mbox->ep_hdl) failed: %d", rc); - - rc=GNI_SmsgInit( - c->recv_mbox->ep_hdl, - &c->recv_mbox->mbox_local_attrs, - &c->recv_mbox->mbox_remote_attrs); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "SmsgInit(c->recv_mbox->ep_hdl) failed: %d", rc); - - log_debug(nnti_debug_level, "new connection recv_mbox->ep_hdl(%llu)", c->recv_mbox->ep_hdl); - - log_debug(nnti_debug_level, "exit"); -} - -static void server_req_queue_remove_client( - nnti_gni_connection_t *c) -{ - int rc; - - log_debug(nnti_debug_level, "enter"); - - rc=GNI_EpUnbind (c->recv_mbox->ep_hdl); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "EpUnbind() failed: %d", rc); - rc=GNI_EpDestroy (c->recv_mbox->ep_hdl); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "EpDestroy() failed: %d", rc); - - rc=GNI_MemDeregister (c->nic_hdl, &c->recv_mbox->mbox_local_attrs.mem_hndl); - if (rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "MemDeregister(1) failed: %d", rc); - - free(c->recv_mbox->mbox_local_attrs.msg_buffer); - free(c->recv_mbox); - - log_debug(nnti_debug_level, "exit"); -} - -static void server_req_queue_destroy( - nnti_gni_request_queue_handle_t *q) -{ - log_debug(nnti_debug_level, "enter"); - - nthread_counter_fini(&q->req_index); - - log_debug(nnti_debug_level, "exit"); -} - - -static inline gni_return_t get_event(gni_cq_handle_t cq, gni_cq_entry_t *ev) -{ - gni_return_t gni_rc; - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_CqGetEvent(cq, ev); - nthread_unlock(&nnti_gni_lock); - if ((gni_rc != GNI_RC_SUCCESS) && (gni_rc != GNI_RC_NOT_DONE)) { - log_error(nnti_debug_level, "CqGetEvent() on cq(%p) failed: %d", cq, gni_rc); - } - return gni_rc; -} - -#define EP_CQ_INDEX 0 -#define MEM_CQ_INDEX 1 -#define INTERRUPT_CQ_INDEX 2 -#define REQ_SEND_EP_CQ_INDEX 3 -#define REQ_SEND_MEM_CQ_INDEX 4 -#define REQ_RECV_EP_CQ_INDEX 5 -#define REQ_RECV_MEM_CQ_INDEX 6 -#define CQ_COUNT 7 - -static NNTI_result_t progress( - int timeout, - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - int rc =0; - gni_return_t gni_rc =GNI_RC_SUCCESS; - NNTI_result_t nnti_rc=NNTI_OK; - - uint32_t which=0; - - nnti_gni_sge_t *gni_sge=NULL; - - gni_cq_handle_t cq_list[CQ_COUNT]; - uint32_t which_cq=0; - gni_cq_entry_t ev_data; - - uint32_t cq_count=CQ_COUNT; - - gni_post_descriptor_t *post_desc_ptr=NULL; - - void *header=NULL; - - long entry_time =trios_get_time_ms(); - long elapsed_time=0; - long gni_timeout =0; - - log_level debug_level=nnti_debug_level; - - static bool in_progress =false; // if true, another thread is already making progress. - bool made_progress=false; - bool interrupted =false; - int8_t wr_complete =FALSE; - - nnti_gni_connection_t *conn=NULL; - uint8_t tag=GNI_SMSG_ANY_TAG; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - - log_debug(debug_level, "enter (timeout=%d)", timeout); - - trios_start_timer(total_time); - - cq_list[EP_CQ_INDEX] =transport_global_data.ep_cq_hdl; - cq_list[MEM_CQ_INDEX] =transport_global_data.mem_cq_hdl; - cq_list[INTERRUPT_CQ_INDEX] =transport_global_data.interrupt_mem_cq_hdl; - cq_list[REQ_SEND_EP_CQ_INDEX] =transport_global_data.req_send_ep_cq_hdl; - cq_list[REQ_SEND_MEM_CQ_INDEX]=transport_global_data.req_send_mem_cq_hdl; - cq_list[REQ_RECV_EP_CQ_INDEX] =transport_global_data.req_recv_ep_cq_hdl; - cq_list[REQ_RECV_MEM_CQ_INDEX]=transport_global_data.req_recv_mem_cq_hdl; - - /* - * Only one thread is allowed to make progress at a time. All others - * wait for the progress maker to finish, then everyone returns at once. - */ - nthread_lock(&nnti_progress_lock); - - wr_complete = is_any_wr_complete(wr_list, wr_count, &which); - - if (!in_progress) { - log_debug(debug_level, "making progress"); - // no other thread is making progress. we'll do it. - in_progress=true; - log_debug(debug_level, "set in_progress=true"); - nthread_unlock(&nnti_progress_lock); - } else { - if (wr_complete == TRUE) { - nthread_unlock(&nnti_progress_lock); - goto cleanup; - } - - // another thread is making progress. we'll wait until they are done. - rc=0; - elapsed_time=0; - if (in_progress) { - if (timeout > 0) { - log_debug(debug_level, "waiting for progress with timeout=%d", timeout-elapsed_time); - // wait for progress or until timeout - rc=nthread_timedwait(&nnti_progress_cond, &nnti_progress_lock, timeout-elapsed_time); - } else if (timeout < 0) { - log_debug(debug_level, "waiting infinitely for progress"); - // infinite wait for progress - rc=nthread_wait(&nnti_progress_cond, &nnti_progress_lock); - } else { - log_debug(debug_level, "waiting for progress with timeout=0. immediate timeout."); - // timeout == 0 and we are not the progress maker. report a timeout. - rc=ETIMEDOUT; - } - elapsed_time = (trios_get_time_ms() - entry_time); - log_debug(debug_level, "rc=%d, elapsed_time=%d", rc, elapsed_time); - } - nthread_unlock(&nnti_progress_lock); - if (rc == ETIMEDOUT) { - log_debug(debug_level, "timed out waiting for progress"); - nnti_rc = NNTI_ETIMEDOUT; - } else if (rc == 0) { - log_debug(debug_level, "someone made progress"); - nnti_rc=NNTI_OK; - } - goto cleanup; - } - - while ((!made_progress) && (!interrupted)) { - - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - nnti_rc=NNTI_ECANCELED; - break; - } - - check_listen_socket_for_new_connections(); - - memset(&ev_data, 0, sizeof(gni_cq_entry_t)); - post_desc_ptr=NULL; - - if (wr_complete == TRUE) { - gni_timeout=0; - } else { - if (timeout-elapsed_time < 0) { - gni_timeout=MIN_TIMEOUT; - } else { - gni_timeout=timeout-elapsed_time; - } - } - - log_debug(nnti_cq_debug_level, "checking for events on any CQ (cq_count=%d ; timeout=%d)", cq_count, gni_timeout); - trios_start_timer(call_time); - gni_rc=GNI_CqVectorMonitor(cq_list, cq_count, gni_timeout, &which_cq); - trios_stop_timer("progress - CqVectorMonitor", call_time); - /* case 1: success */ - if (gni_rc == GNI_RC_SUCCESS) { - print_cq_event(&ev_data, false); - - switch (which_cq) { - - case INTERRUPT_CQ_INDEX: - if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { - log_debug(nnti_debug_level, "CqVectorMonitor() interrupted by NNTI_gni_interrupt"); - interrupted=true; - nnti_rc = NNTI_EINTR; - } - continue; - - case REQ_RECV_EP_CQ_INDEX: - if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { - log_debug(nnti_debug_level, "CqVectorMonitor(req_recv_ep_cq_hdl) SMSG implicit credit event received at receiver (I sent credits)"); - } - continue; - - case REQ_SEND_MEM_CQ_INDEX: - if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { - log_debug(nnti_debug_level, "CqVectorMonitor(req_send_mem_cq_hdl) SMSG implicit credit event received at sender (I received credits)"); - } - continue; - - case REQ_RECV_MEM_CQ_INDEX: -another_event: - if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { - conn=get_conn_instance((uint32_t)gni_cq_get_inst_id(ev_data)); - assert(conn); - - tag=GNI_SMSG_ANY_TAG; - - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_SmsgGetNextWTag(conn->recv_mbox->ep_hdl, &header, &tag); - nthread_unlock(&nnti_gni_lock); - if (gni_rc == GNI_RC_SUCCESS) { - if (tag == NNTI_SMSG_TAG_CREDIT) { - nnti_gni_credit_msg_t *credit=(nnti_gni_credit_msg_t *)header; - log_debug(nnti_debug_level, "SmsgGetNextWTag(recv_mbox->ep_hdl) SMSG explicit credit event received at receiver: credit_return_msg.inst_id=%llu", credit->inst_id); - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_SmsgRelease(conn->recv_mbox->ep_hdl); - nthread_unlock(&nnti_gni_lock); - if (gni_rc!=GNI_RC_SUCCESS) log_error(nnti_debug_level, "SmsgRelease(request) failed: %d", gni_rc); - } else if (tag == NNTI_SMSG_TAG_REQUEST) { - log_debug(nnti_debug_level, "SmsgGetNextWTag(recv_mbox->ep_hdl) SMSG request received"); - - gni_sge=decode_sge(&ev_data, cq_list[which_cq]); - if (gni_sge) { - rc=process_event(gni_sge->gni_wr->reg_buf, gni_sge, header, cq_list[which_cq], &ev_data, post_desc_ptr); - if (rc != NNTI_OK) { - log_error(nnti_debug_level, "process_event() failed: %d", rc); - continue; - } - } - - made_progress=true; - - } else { - log_debug(nnti_debug_level, "SmsgGetNextWTag(recv_mbox->ep_hdl) SMSG unknown tag: %d", tag); - } - } else if (gni_rc == GNI_RC_NO_MATCH) { -// continue; - } else if (gni_rc == GNI_RC_NOT_DONE) { -// continue; - } else { - log_debug(nnti_debug_level, "SmsgGetNextWTag(recv_mbox->ep_hdl) failed: %d", gni_rc); -// continue; - } - log_debug(nnti_debug_level, "goto another_event"); - goto another_event; - } - break; - - case REQ_SEND_EP_CQ_INDEX: - if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { - log_debug(nnti_debug_level, "CqVectorMonitor(req_send_ep_cq_hdl) SMSG send complete event received at sender"); - - gni_sge=decode_sge(&ev_data, cq_list[which_cq]); - if (gni_sge) { - rc=process_event(gni_sge->gni_wr->reg_buf, gni_sge, header, cq_list[which_cq], &ev_data, post_desc_ptr); - if (rc != NNTI_OK) { - log_error(nnti_debug_level, "process_event() failed: %d", rc); - continue; - } - } - - made_progress=true; - } - break; - - default: - if (get_event(cq_list[which_cq], &ev_data) == GNI_RC_SUCCESS) { - if (gni_cq_get_source(ev_data) == 1) { - log_debug(nnti_debug_level, "calling GetComplete(progress)"); - nthread_lock(&nnti_gni_lock); - gni_rc=GNI_GetCompleted (cq_list[which_cq], ev_data, &post_desc_ptr); - nthread_unlock(&nnti_gni_lock); - if (gni_rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "GetCompleted(progress post_desc_ptr(%p)) failed: %d", post_desc_ptr, gni_rc); - } else { - log_debug(nnti_debug_level, "GetCompleted(progress post_desc_ptr(%p)) success", post_desc_ptr); - } - print_post_desc(post_desc_ptr); - } - log_debug(nnti_debug_level, "CqVectorMonitor() RDMA event received"); - - gni_sge=decode_sge(&ev_data, cq_list[which_cq]); - if (gni_sge) { - rc=process_event(gni_sge->gni_wr->reg_buf, gni_sge, header, cq_list[which_cq], &ev_data, post_desc_ptr); - if (rc != NNTI_OK) { - log_error(nnti_debug_level, "process_event() failed: %d", rc); - continue; - } - } - - made_progress=true; - } - break; - } - - nnti_rc = NNTI_OK; - } - /* case 2: timed out */ - else if ((gni_rc==GNI_RC_TIMEOUT) || (gni_rc==GNI_RC_NOT_DONE)) { - if (wr_complete == TRUE) { - nnti_rc=NNTI_OK; - break; - } - - elapsed_time = (trios_get_time_ms() - entry_time); - - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout >= 0) && (elapsed_time >= timeout)) || trios_exit_now()) { - log_debug(nnti_debug_level, "CqVectorMonitor timed out...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - /* continue if the timeout has not expired */ - log_debug(nnti_event_debug_level, "CqVectorMonitor timedout...retrying...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now()); - - continue; - } - /* case 3: failure */ - else { - char errstr[1024]; - uint32_t recoverable=0; - errstr[0]='\0'; - GNI_CqErrorStr(ev_data, errstr, 1023); - GNI_CqErrorRecoverable(ev_data, &recoverable); - - log_error(nnti_debug_level, "CqVectorMonitor failed (gni_rc=%d) (recoverable=%llu) : %s", - gni_rc, (uint64_t)recoverable, errstr); - print_cq_event(&ev_data, false); - - gni_sge=decode_sge(&ev_data, cq_list[which_cq]); - process_event(gni_sge->gni_wr->reg_buf, gni_sge, NULL, cq_list[which_cq], &ev_data, post_desc_ptr); - - nnti_rc = NNTI_EIO; - - made_progress=true; - } - } - - nthread_lock(&nnti_progress_lock); - in_progress=false; - log_debug(debug_level, "set in_progress=false"); - nthread_broadcast(&nnti_progress_cond); - log_debug(debug_level, "broadcasted on nnti_progress_cond"); - nthread_unlock(&nnti_progress_lock); - -cleanup: - trios_stop_timer("progress", total_time); - - log_debug(debug_level, "exit"); - - return(nnti_rc); -} - - - -#define LISTEN_IFACE_BASENAME "ipog" -static NNTI_result_t get_ipaddr( - char *ipaddr, - int maxlen) -{ - struct ifaddrs * ifAddrStruct=NULL; - struct ifaddrs * ifa=NULL; - void * tmpAddrPtr=NULL; - - getifaddrs(&ifAddrStruct); - for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) { - ipaddr[0]='\0'; - if (ifa ->ifa_addr->sa_family==AF_INET) { // check it is IP4 - // is a valid IP4 Address - tmpAddrPtr=&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr; - log_debug(nnti_debug_level, "checking iface (IPv4) name (%s)", ifa->ifa_name); - inet_ntop(AF_INET, tmpAddrPtr, ipaddr, maxlen); - log_debug(nnti_debug_level, "hostname(%s) has IP Address %s", ifa->ifa_name, ipaddr); - if (0==strncmp(ifa->ifa_name, LISTEN_IFACE_BASENAME, strlen(LISTEN_IFACE_BASENAME))) { - log_debug(nnti_debug_level, "hostname(%s) matches", ifa->ifa_name); - break; - } - } else if (ifa->ifa_addr->sa_family==AF_INET6) { // check it is IP6 - // is a valid IP6 Address - tmpAddrPtr=&((struct sockaddr_in *)ifa->ifa_addr)->sin_addr; - log_debug(nnti_debug_level, "checking iface (IPv6) name (%s)", ifa->ifa_name); - inet_ntop(AF_INET6, tmpAddrPtr, ipaddr, maxlen); - log_debug(nnti_debug_level, "hostname(%s) has IP Address %s", ifa->ifa_name, ipaddr); - if (0==strncmp(ifa->ifa_name, LISTEN_IFACE_BASENAME, strlen(LISTEN_IFACE_BASENAME))) { - log_debug(nnti_debug_level, "hostname(%s) matches", ifa->ifa_name); - break; - } - } - } - if (ifAddrStruct!=NULL) freeifaddrs(ifAddrStruct); - - log_debug(nnti_debug_level, "ipaddr(%s)", ipaddr); - if (ipaddr[0]=='\0') { - return NNTI_ENOENT; - } else { - return NNTI_OK; - } -} - - -static void config_init(nnti_gni_config_t *c) -{ - c->min_atomics_vars =512; - c->use_alps_ptag =true; - c->use_wr_pool =false; - c->wr_pool_initial_size =50; - c->wr_pool_max_size =1000; - c->wr_pool_create_if_empty =false; - c->use_rdma_target_ack =false; - c->use_rdma_events =false; - c->use_rdma_fence =false; - c->pi_ordering =PI_ORDERING_DEFAULT; /* DEFAULT, STRICT, RELAXED */ - c->rdma_mode =RDMA_MIXED; /* FMA, BTE, MIXED, CROSSOVER */ - c->fma_bte_crossover_size =4096; - c->max_timeout_ms =200; - c->queue_elements_per_connection=32; -} - -static void config_get_from_env(nnti_gni_config_t *c) -{ - char *env_str=NULL; - - if ((env_str=getenv("TRIOS_NNTI_USE_ALPS_PTAG")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_alps_ptag to TRUE"); - c->use_alps_ptag=true; - } else { - log_debug(nnti_debug_level, "setting c->use_alps_ptag to FALSE"); - c->use_alps_ptag=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_ALPS_PTAG is undefined. using c->use_alps_ptag default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_WR_POOL")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_wr_pool to TRUE"); - c->use_wr_pool=true; - } else { - log_debug(nnti_debug_level, "setting c->use_wr_pool to FALSE"); - c->use_wr_pool=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_WR_POOL is undefined. using c->use_wr_pool default"); - } - if ((env_str=getenv("TRIOS_NNTI_WR_POOL_INITIAL_SIZE")) != NULL) { - errno=0; - uint32_t initial_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->wr_pool_initial_size to %lu", initial_size); - c->wr_pool_initial_size=initial_size; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_WR_POOL_INITIAL_SIZE value conversion failed (%s). using c->wr_pool_initial_size default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_WR_POOL_INITIAL_SIZE is undefined. using c->wr_pool_initial_size default"); - } - if ((env_str=getenv("TRIOS_NNTI_WR_POOL_MAX_SIZE")) != NULL) { - errno=0; - uint32_t max_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->wr_pool_max_size to %lu", max_size); - c->wr_pool_max_size=max_size; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_WR_POOL_MAX_SIZE value conversion failed (%s). using c->wr_pool_max_size default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_WR_POOL_MAX_SIZE is undefined. using c->wr_pool_max_size default"); - } - if ((env_str=getenv("TRIOS_NNTI_WR_POOL_CREATE_IF_EMPTY")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->wr_pool_create_if_empty to TRUE"); - c->wr_pool_create_if_empty=true; - } else { - log_debug(nnti_debug_level, "setting c->wr_pool_create_if_empty to FALSE"); - c->wr_pool_create_if_empty=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_WR_POOL_CREATE_IF_EMPTY is undefined. using c->wr_pool_create_if_empty default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_RDMA_TARGET_ACK")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_rdma_target_ack to TRUE"); - c->use_rdma_target_ack=true; - } else { - log_debug(nnti_debug_level, "setting c->use_rdma_target_ack to FALSE"); - c->use_rdma_target_ack=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_RDMA_TARGET_ACK is undefined. using c->use_rdma_target_ack default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_RDMA_EVENTS")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_rdma_events to TRUE"); - c->use_rdma_events=true; - } else { - log_debug(nnti_debug_level, "setting c->use_rdma_events to FALSE"); - c->use_rdma_events=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_RDMA_EVENTS is undefined. using c->use_rdma_events default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_RDMA_FENCE")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_rdma_fence to TRUE"); - c->use_rdma_fence=true; - } else { - log_debug(nnti_debug_level, "setting c->use_rdma_fence to FALSE"); - c->use_rdma_fence=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_RDMA_FENCE is undefined. using c->use_rdma_fence default"); - } - if ((env_str=getenv("TRIOS_NNTI_PI_ORDERING")) != NULL) { - if (!strcasecmp(env_str, "PI_ORDERING_DEFAULT")) { - log_debug(nnti_debug_level, "setting c->pi_ordering to PI_ORDERING_DEFAULT"); - c->pi_ordering=PI_ORDERING_DEFAULT; - } else if (!strcasecmp(env_str, "PI_ORDERING_STRICT")) { - log_debug(nnti_debug_level, "setting c->pi_ordering to PI_ORDERING_STRICT"); - c->pi_ordering=PI_ORDERING_STRICT; - } else if (!strcasecmp(env_str, "PI_ORDERING_RELAXED")) { - log_debug(nnti_debug_level, "setting c->pi_ordering to PI_ORDERING_RELAXED"); - c->pi_ordering=PI_ORDERING_RELAXED; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_PI_ORDERING is undefined. using c->pi_ordering default"); - } - if ((env_str=getenv("TRIOS_NNTI_RDMA_MODE")) != NULL) { - if (!strcasecmp(env_str, "RDMA_FMA")) { - log_debug(nnti_debug_level, "setting c->rdma_mode to RDMA_FMA"); - c->rdma_mode=RDMA_FMA; - } else if (!strcasecmp(env_str, "RDMA_BTE")) { - log_debug(nnti_debug_level, "setting c->rdma_mode to RDMA_BTE"); - c->rdma_mode=RDMA_BTE; - } else if (!strcasecmp(env_str, "RDMA_MIXED")) { - log_debug(nnti_debug_level, "setting c->rdma_mode to RDMA_MIXED"); - c->rdma_mode=RDMA_MIXED; - } else if (!strcasecmp(env_str, "RDMA_CROSSOVER")) { - log_debug(nnti_debug_level, "setting c->rdma_mode to RDMA_CROSSOVER"); - c->rdma_mode=RDMA_CROSSOVER; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_RDMA_MODE is undefined. using c->rdma_mode default"); - } - if ((env_str=getenv("TRIOS_NNTI_FMA_BTE_CROSSOVER_SIZE")) != NULL) { - errno=0; - uint32_t crossover_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->fma_bte_crossover_size to %lu", crossover_size); - c->fma_bte_crossover_size=crossover_size; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_FMA_BTE_CROSSOVER_SIZE value conversion failed (%s). using c->fma_bte_crossover_size default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_FMA_BTE_CROSSOVER_SIZE is undefined. using c->fma_bte_crossover_size default"); - } - if ((env_str=getenv("TRIOS_NNTI_MAX_TIMEOUT_MS")) != NULL) { - errno=0; - int32_t max_timeout_ms=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->max_timeout_ms to %lu", max_timeout_ms); - c->max_timeout_ms=max_timeout_ms; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MAX_TIMEOUT_MS value conversion failed (%s). using c->max_timeout_ms default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MAX_TIMEOUT_MS is undefined. using c->max_timeout_ms default"); - } - if ((env_str=getenv("TRIOS_NNTI_MIN_ATOMIC_VARS")) != NULL) { - errno=0; - uint32_t min_vars=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->min_atomics_vars to %lu", min_vars); - c->min_atomics_vars=min_vars; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MIN_ATOMIC_VARS value conversion failed (%s). using c->min_atomics_vars default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MIN_ATOMIC_VARS is undefined. using c->min_atomics_vars default"); - } - if ((env_str=getenv("TRIOS_NNTI_QUEUE_ELEMENTS_PER_CONNECTION")) != NULL) { - errno=0; - uint32_t qelem=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->queue_elements_per_connection to %lu", qelem); - c->queue_elements_per_connection=qelem; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_QUEUE_ELEMENTS_PER_CONNECTION value conversion failed (%s). using c->queue_elements_per_connection default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_QUEUE_ELEMENTS_PER_CONNECTION is undefined. using c->queue_elements_per_connection default"); - } -} - -inline int DEQUEUE_POST_DESCRIPTOR( - gni_cq_handle_t cq_hdl, - gni_cq_entry_t *ev_data, - gni_post_descriptor_t **post_desc_ptr) -{ - int rc=GNI_RC_SUCCESS; - - *post_desc_ptr=NULL; - - log_debug(nnti_debug_level, "calling GetComplete(DEQUEUE_POST_DESCRIPTOR)"); - rc=GNI_GetCompleted (cq_hdl, *ev_data, post_desc_ptr); - if (rc!=GNI_RC_SUCCESS) { - log_error(nnti_debug_level, "GetCompleted(DEQUEUE_POST_DESCRIPTOR post_desc_ptr(%p)) failed: %d", *post_desc_ptr, rc); - } else { - log_debug(nnti_debug_level, "GetCompleted(DEQUEUE_POST_DESCRIPTOR post_desc_ptr(%p)) success", *post_desc_ptr); - } - print_post_desc(*post_desc_ptr); - - return(rc); -} diff --git a/packages/trios/libraries/nessie/nnti/nnti_gni.h b/packages/trios/libraries/nessie/nnti/nnti_gni.h deleted file mode 100644 index e8fbf760334a..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_gni.h +++ /dev/null @@ -1,228 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * nnti_gni.h - * - * Created on: Jan 13, 2011 - * Author: thkorde - */ - -#ifndef NNTI_GNI_H_ -#define NNTI_GNI_H_ - - -#include "Trios_nnti.h" -#include "nnti_internal.h" - - - -#ifdef __cplusplus -extern "C" { -#endif - -#define GNI_SET_MATCH_ANY(p) \ -(p)->peer.transport_id = NNTI_TRANSPORT_GEMINI; \ -(p)->peer.NNTI_remote_process_t_u.gni.addr = 0; \ -(p)->peer.NNTI_remote_process_t_u.gni.port = 0; - - -NNTI_result_t NNTI_gni_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_gni_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -NNTI_result_t NNTI_gni_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_gni_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_gni_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_gni_free ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_gni_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_gni_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_gni_unregister_memory ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_gni_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context); - -NNTI_result_t NNTI_gni_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value); - -NNTI_result_t NNTI_gni_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_clear_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_destroy_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_cancel ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_gni_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -NNTI_result_t NNTI_gni_interrupt ( - const NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_gni_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status); - -NNTI_result_t NNTI_gni_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -NNTI_result_t NNTI_gni_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status); - -NNTI_result_t NNTI_gni_fini ( - const NNTI_transport_t *trans_hdl); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_GNI_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/nnti_ib.cpp b/packages/trios/libraries/nessie/nnti/nnti_ib.cpp deleted file mode 100644 index 964bacae6ad9..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_ib.cpp +++ /dev/null @@ -1,7065 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_ib.c - * - * Created on: Jan 13, 2011 - * Author: thkorde - */ - -#include "Trios_config.h" - -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#ifdef HAVE_TRIOS_HPCTOOLKIT -#include -#define SAMPLING_IS_ACTIVE() hpctoolkit_sampling_is_active() -#define SAMPLING_STOP() hpctoolkit_sampling_stop() -#define SAMPLING_START() hpctoolkit_sampling_start() -#else -#define SAMPLING_IS_ACTIVE() 0 -#define SAMPLING_STOP() -#define SAMPLING_START() -#endif - -#include "nnti_ib.h" -#include "nnti_utils.h" - - - -//#define USE_WR_POOL -//#undef USE_WR_POOL -// -// -///* if undefined, the ACK message is NOT sent to the RDMA target when -// * the RDMA op is complete. this creates one-sided semantics for RDMA -// * ops. in this mode, the target has no idea when the RDMA op is -// * complete and what data was addressed. NNTI_wait() returns NNTI_EINVAL -// * if passed a target buffer. -// */ -//#undef USE_RDMA_TARGET_ACK -///* if defined, the RDMA initiator will send an ACK message to the RDMA -// * target when the RDMA op is complete. the target process must wait -// * on the target buffer in order to get the ACK. this creates two-sided -// * semantics for RDMA ops. in this mode, when the wait returns the -// * the RDMA op is complete and status indicates what data was addressed. -// */ -//#define USE_RDMA_TARGET_ACK - - - -typedef struct { - - uint32_t min_atomics_vars; - - bool use_wr_pool; - - bool use_rdma_target_ack; - - bool use_mlock; - bool use_memset; - - bool drop_if_full_queue; - -} nnti_ib_config; - - -/** - * These states are used to signal events between the completion handler - * and the main client or server thread. - * - * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, - * and RDMA_WRITE_COMPLETE for each ping. - */ -typedef enum { - IDLE = 1, - CONNECT_REQUEST, - ADDR_RESOLVED, - ROUTE_RESOLVED, - CONNECTED, - DISCONNECTED, - ERROR -} ib_connection_state; - -#define IB_OP_PUT_INITIATOR 1 -#define IB_OP_GET_INITIATOR 2 -#define IB_OP_PUT_TARGET 3 -#define IB_OP_GET_TARGET 4 -#define IB_OP_SEND_REQUEST 5 -#define IB_OP_SEND_BUFFER 6 -#define IB_OP_NEW_REQUEST 7 -#define IB_OP_RECEIVE 8 -#define IB_OP_FETCH_ADD 9 -#define IB_OP_COMPARE_SWAP 10 - -typedef enum { - NNTI_IB_WR_STATE_RESET, // this work request is idle - NNTI_IB_WR_STATE_POSTED, // target is ready to receive data - NNTI_IB_WR_STATE_STARTED, // initiator has posted the sq_wr descriptor - NNTI_IB_WR_STATE_ATTACHED, // target has attached this work request to an NNTI_work_request_t - NNTI_IB_WR_STATE_CANCELING, // NNTI_cancel() called, but not NNTI_wait() - NNTI_IB_WR_STATE_RDMA_COMPLETE, // RDMA op complete - NNTI_IB_WR_STATE_ACK_COMPLETE, // work completion transfer complete - NNTI_IB_WR_STATE_WAIT_COMPLETE, // NNTI_wait() called and completed successfully - NNTI_IB_WR_STATE_ERROR // something went wrong. check wr->result -} nnti_ib_wr_state_t; - -#define SRQ_DEPTH 20480 -#define CQ_DEPTH 20480 - -typedef struct { - struct ibv_qp *qp; - uint32_t qpn; - uint32_t peer_qpn; -} conn_qp; -typedef struct { - NNTI_peer_t peer; - char *peer_name; - NNTI_ip_addr peer_addr; - NNTI_tcp_port peer_port; - uint16_t peer_lid; - uint32_t peer_req_qpn; - - conn_qp req_qp; - conn_qp data_qp; - - uint32_t atomics_rkey; - uint64_t atomics_addr; - - ib_connection_state state; - - int8_t disconnect_requested; -} ib_connection; - -typedef struct { - uint32_t op; - uint64_t offset; - uint64_t length; -} ib_rdma_ack; - -typedef struct { - nthread_lock_t lock; - - NNTI_work_request_t *nnti_wr; - - nnti_ib_wr_state_t state; - - uint32_t key; - - NNTI_buffer_t *reg_buf; - ib_connection *conn; - - struct ibv_send_wr sq_wr; - struct ibv_send_wr *sq_wr_list; - uint32_t sq_wr_count; - uint32_t sq_wr_completed_count; - struct ibv_recv_wr rq_wr; - struct ibv_sge sge; - struct ibv_sge *sge_list; - uint32_t sge_count; - - struct ibv_comp_channel *comp_channel; - struct ibv_cq *cq; - struct ibv_qp *qp; - uint32_t qpn; - uint32_t peer_qpn; - - struct ibv_send_wr ack_sq_wr; - struct ibv_recv_wr ack_rq_wr; - struct ibv_sge ack_sge; - struct ibv_mr *ack_mr; - ib_rdma_ack ack; - - /* this is a copy of the last work completion that arrived for this buffer */ - struct ibv_wc last_wc; - - uint8_t last_op; - uint64_t offset; - uint64_t length; - -} ib_work_request; - -typedef std::deque wr_queue_t; -typedef std::deque::iterator wr_queue_iter_t; - -typedef struct { - struct ibv_mr *mr; - struct ibv_mr **mr_list; - uint32_t mr_count; - wr_queue_t wr_queue; - nthread_lock_t wr_queue_lock; - uint32_t ref_count; -} ib_memory_handle; - -typedef struct { - NNTI_buffer_t *reg_buf; - - char *req_buffer; /* incoming queue */ - int req_count; /* incoming queue */ - int req_size; /* each message is no larger than req_size */ - - uint32_t req_received; -} ib_request_queue_handle; - -typedef struct { - NNTI_peer_t me; - - struct ibv_device *dev; - struct ibv_context *ctx; - struct ibv_pd *pd; - - uint32_t cqe_count; - uint32_t srq_count; - uint32_t sge_count; - uint32_t qp_count; - - struct ibv_comp_channel *req_comp_channel; - struct ibv_cq *req_cq; - struct ibv_srq *req_srq; - - struct ibv_comp_channel *data_comp_channel; - struct ibv_cq *data_cq; - struct ibv_srq *data_srq; - - uint32_t req_srq_count; - uint32_t data_srq_count; - - uint16_t nic_lid; - int nic_port; - - int listen_sock; - char listen_name[NNTI_HOSTNAME_LEN]; - uint32_t listen_addr; /* in NBO */ - uint16_t listen_port; /* in NBO */ - - int interrupt_pipe[2]; - - int64_t *atomics; - struct ibv_mr *atomics_mr; - nthread_lock_t atomics_lock; - - ib_request_queue_handle req_queue; -} ib_transport_global; - - - - -static nthread_lock_t nnti_ib_lock; -static nthread_lock_t nnti_progress_lock; -static nthread_cond_t nnti_progress_cond; - -static void *aligned_malloc( - size_t size); -static struct ibv_mr *register_memory_segment( - void *buf, - uint64_t len, - enum ibv_access_flags access); -static int unregister_memory_segment( - struct ibv_mr *mr); -static int register_ack( - ib_work_request *ib_wr); -static int unregister_ack( - ib_work_request *ib_wr); -static void send_ack ( - ib_work_request *ib_wr); -static NNTI_result_t setup_data_channel(void); -static NNTI_result_t setup_request_channel(void); -static NNTI_result_t setup_interrupt_pipe(void); -static NNTI_result_t setup_atomics(void); -static ib_work_request *decode_work_request( - const struct ibv_wc *wc); -static int cancel_wr( - ib_work_request *ib_wr); -static int process_event( - ib_work_request *ib_wr, - const struct ibv_wc *wc); -static NNTI_result_t post_recv_work_request( - NNTI_buffer_t *reg_buf, - int64_t wr_id, - uint64_t offset); -static NNTI_result_t post_ack_recv_work_request( - NNTI_buffer_t *reg_buf); -static NNTI_result_t repost_recv_work_request( - NNTI_work_request_t *wr, - ib_work_request *ib_wr); -static NNTI_result_t repost_ack_recv_work_request( - NNTI_work_request_t *wr, - ib_work_request *ib_wr); -static int8_t is_wr_canceling( - ib_work_request *ib_wr); -static int8_t is_wr_complete( - ib_work_request *ib_wr); -static int8_t is_any_wr_complete( - ib_work_request **wr_list, - const uint32_t wr_count, - uint32_t *which); -static int8_t is_all_wr_complete( - ib_work_request **wr_list, - const uint32_t wr_count); -static int8_t is_wr_complete( - NNTI_work_request_t *wr); -static int8_t is_any_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - uint32_t *which); -static int8_t is_all_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); -static void create_status( - NNTI_work_request_t *wr, - ib_work_request *ib_wr, - NNTI_result_t nnti_rc, - NNTI_status_t *status); -static void create_peer( - NNTI_peer_t *peer, - char *name, - NNTI_ip_addr addr, - NNTI_tcp_port port); -static void copy_peer( - NNTI_peer_t *dest, - NNTI_peer_t *src); -static int init_server_listen_socket(void); -static NNTI_result_t check_listen_socket_for_new_connections(); -static struct ibv_device *get_ib_device(void); -static void get_qp_state(struct ibv_qp *qp); -static void print_all_qp_state(void); -static int tcp_read(int sock, void *incoming, size_t len); -static int tcp_write(int sock, const void *outgoing, size_t len); -static int tcp_exchange(int sock, int is_server, void *incoming, void *outgoing, size_t len); -static void transition_connection_to_ready( - int sock, - ib_connection *conn); -static void transition_qp_from_reset_to_ready( - struct ibv_qp *qp, - uint32_t peer_qpn, - int peer_lid, - int min_rnr_timer, - int ack_timeout, - int retry_count); -static void transition_qp_from_error_to_ready( - struct ibv_qp *qp, - uint32_t peer_qpn, - int peer_lid, - int min_rnr_timer, - int ack_timeout, - int retry_count); -static void transition_connection_to_error( - ib_connection *conn); -static void transition_qp_to_error( - struct ibv_qp *qp, - uint32_t peer_qpn, - int peer_lid); -static NNTI_result_t init_connection( - ib_connection **conn, - const int sock, - const int is_server); -static void close_connection(ib_connection *c); -static void print_wc( - const struct ibv_wc *wc, - bool force); -static void print_ib_conn(ib_connection *c); -//static void print_qpn_map(void); -//static void print_peer_map(void); -static NNTI_result_t insert_conn_peer(const NNTI_peer_t *peer, ib_connection *conn); -static NNTI_result_t insert_conn_qpn(const NNTI_qp_num qpn, ib_connection *conn); -static ib_connection *get_conn_peer(const NNTI_peer_t *peer); -static ib_connection *get_conn_qpn(const NNTI_qp_num qpn); -static NNTI_peer_t *get_peer_by_url(const char *url); -static ib_connection *del_conn_peer(const NNTI_peer_t *peer); -static ib_connection *del_conn_qpn(const NNTI_qp_num qpn); - -static NNTI_result_t insert_buf_bufhash(NNTI_buffer_t *buf); -static NNTI_buffer_t *get_buf_bufhash(const uint32_t bufhash); -static NNTI_buffer_t *del_buf_bufhash(NNTI_buffer_t *buf); -//static void print_bufhash_map(void); - -static NNTI_result_t wr_pool_init(uint32_t pool_size); -static ib_work_request *wr_pool_rdma_pop(void); -static ib_work_request *wr_pool_sendrecv_pop(void); -static void wr_pool_rdma_push(ib_work_request *ib_wr); -static void wr_pool_sendrecv_push(ib_work_request *ib_wr); -static NNTI_result_t wr_pool_fini(void); - -static void close_all_conn(void); - -static NNTI_result_t poll_all( - int timeout); -static NNTI_result_t progress( - int timeout, - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -static void config_init( - nnti_ib_config *c); -static void config_get_from_env( - nnti_ib_config *c); - -//static void print_wr(ib_work_request *ib_wr); -//static void print_xfer_buf(void *buf, uint32_t size); -//static void print_ack_buf(ib_rdma_ack *ack); - -#define IB_MEM_HDL(b) ((ib_memory_handle *)((b)->transport_private)) -#define IB_WORK_REQUEST(ib_wr) ((ib_work_request *)((ib_wr)->transport_private)) - -static bool ib_initialized=false; - -static ib_transport_global transport_global_data; -static const int MIN_TIMEOUT = 0; /* in milliseconds */ - - -/** - * This custom key is used to look up existing connections. - */ -struct addrport_key { - NNTI_ip_addr addr; /* part1 of a compound key */ - NNTI_tcp_port port; /* part2 of a compound key */ - - // Need this operators for the hash map - bool operator<(const addrport_key &key1) const { - if (addr < key1.addr) { - return true; - } else if (addr == key1.addr) { - if (port < key1.port) { - return true; - } - } - return false; - } - bool operator>(const addrport_key &key1) const { - if (addr > key1.addr) { - return true; - } else if (addr == key1.addr) { - if (port > key1.port) { - return true; - } - } - return false; - } -}; - - -/* Thomas Wang's 64 bit to 32 bit Hash Function (http://www.concentric.net/~ttwang/tech/inthash.htm) */ -static uint32_t hash6432shift(uint64_t key) -{ - key = (~key) + (key << 18); // key = (key << 18) - key - 1; - key = key ^ (key >> 31); - key = key * 21; // key = (key + (key << 2)) + (key << 4); - key = key ^ (key >> 11); - key = key + (key << 6); - key = key ^ (key >> 22); - return (uint32_t)key; -} - -/* - * We need a couple of maps to keep track of connections. Servers need to find - * connections by QP number when requests arrive. Clients need to find connections - * by peer address and port. Setup those maps here. - */ -static std::map connections_by_peer; -typedef std::map::iterator conn_by_peer_iter_t; -typedef std::pair conn_by_peer_t; -static nthread_lock_t nnti_conn_peer_lock; - -static std::map connections_by_qpn; -typedef std::map::iterator conn_by_qpn_iter_t; -typedef std::pair conn_by_qpn_t; -static nthread_lock_t nnti_conn_qpn_lock; - -static std::map buffers_by_bufhash; -typedef std::map::iterator buf_by_bufhash_iter_t; -typedef std::pair buf_by_bufhash_t; -static nthread_lock_t nnti_buf_bufhash_lock; - -typedef std::deque wr_pool_t; -typedef std::deque::iterator wr_pool_iter_t; -static nthread_lock_t nnti_wr_pool_lock; - -typedef uint32_t wr_key_t; -static std::map wrmap; -typedef std::map::iterator wrmap_iter_t; -//typedef std::pair wrmap_pair_t; -static nthread_lock_t nnti_wrmap_lock; -static nthread_counter_t nnti_wrmap_counter; - -static wr_pool_t rdma_wr_pool; -static wr_pool_t sendrecv_wr_pool; - - -static nnti_ib_config config; - - -/* ---------------- Wrappers to protect HPCToolkit issues ---------- */ -static -void ibv_ack_cq_events_wrapper(struct ibv_cq *cq, unsigned int nevents) -{ - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - ibv_ack_cq_events(cq, nevents); - if (sampling) SAMPLING_START(); - - return; -} -static -struct ibv_pd *ibv_alloc_pd_wrapper(struct ibv_context *context) -{ - struct ibv_pd *pd=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - pd=ibv_alloc_pd(context); - if (sampling) SAMPLING_START(); - - return pd; -} -static -int ibv_close_device_wrapper(struct ibv_context *context) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_close_device(context); - if (sampling) SAMPLING_START(); - - return rc; -} -static -struct ibv_comp_channel *ibv_create_comp_channel_wrapper(struct ibv_context *context) -{ - struct ibv_comp_channel *channel=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - channel=ibv_create_comp_channel(context); - if (sampling) SAMPLING_START(); - - return channel; -} -static -struct ibv_cq *ibv_create_cq_wrapper(struct ibv_context *context, int cqe, - void *cq_context, - struct ibv_comp_channel *channel, - int comp_vector) -{ - struct ibv_cq *cq=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - cq=ibv_create_cq(context, cqe, cq_context, channel, comp_vector); - if (sampling) SAMPLING_START(); - - return cq; -} -static -struct ibv_qp *ibv_create_qp_wrapper(struct ibv_pd *pd, - struct ibv_qp_init_attr *qp_init_attr) -{ - struct ibv_qp *qp=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - qp=ibv_create_qp(pd, qp_init_attr); - if (sampling) SAMPLING_START(); - - return qp; -} -static -struct ibv_srq *ibv_create_srq_wrapper(struct ibv_pd *pd, - struct ibv_srq_init_attr *srq_init_attr) -{ - struct ibv_srq *srq=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - srq=ibv_create_srq(pd, srq_init_attr); - if (sampling) SAMPLING_START(); - - return srq; -} -static -int ibv_dealloc_pd_wrapper(struct ibv_pd *pd) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_dealloc_pd(pd); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_dereg_mr_wrapper(struct ibv_mr *mr) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_dereg_mr(mr); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_destroy_comp_channel_wrapper( - struct ibv_comp_channel *channel) -{ - int rc; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_destroy_comp_channel(channel); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_destroy_cq_wrapper( - struct ibv_cq *cq) -{ - int rc; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_destroy_cq(cq); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_destroy_qp_wrapper(struct ibv_qp *qp) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_destroy_qp(qp); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_destroy_srq_wrapper( - struct ibv_srq *srq) -{ - int rc; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_destroy_srq(srq); - if (sampling) SAMPLING_START(); - - return rc; -} -static -void ibv_free_device_list_wrapper(struct ibv_device **list) -{ - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - ibv_free_device_list(list); - if (sampling) SAMPLING_START(); - - return; -} -static -int ibv_get_cq_event_wrapper(struct ibv_comp_channel *channel, - struct ibv_cq **cq, - void **cq_context) -{ - int rc; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_get_cq_event(channel, cq, cq_context); - if (sampling) SAMPLING_START(); - - return rc; -} -static -struct ibv_device **ibv_get_device_list_wrapper(int *num_devices) -{ - struct ibv_device **dev=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - dev=ibv_get_device_list(num_devices); - if (sampling) SAMPLING_START(); - - return dev; -} -static -int ibv_modify_qp_wrapper(struct ibv_qp *qp, - struct ibv_qp_attr *attr, - int attr_mask) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_modify_qp(qp, attr, attr_mask); - if (sampling) SAMPLING_START(); - - return rc; -} -static -struct ibv_context *ibv_open_device_wrapper(struct ibv_device *device) -{ - struct ibv_context *context=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - context=ibv_open_device(device); - if (sampling) SAMPLING_START(); - - return context; -} -static -int ibv_poll_cq_wrapper(struct ibv_cq *cq, int num_entries, struct ibv_wc *wc) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_poll_cq(cq, num_entries, wc); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_post_send_wrapper(struct ibv_qp *qp, - struct ibv_send_wr *ib_wr, - struct ibv_send_wr **bad_wr) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_post_send(qp, ib_wr, bad_wr); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_post_srq_recv_wrapper(struct ibv_srq *srq, - struct ibv_recv_wr *recv_wr, - struct ibv_recv_wr **bad_recv_wr) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_post_srq_recv(srq, recv_wr, bad_recv_wr); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_query_device_wrapper(struct ibv_context *context, - struct ibv_device_attr *device_attr) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_query_device(context, device_attr); - if (sampling) SAMPLING_START(); - - return rc; -} -static -int ibv_query_port_wrapper(struct ibv_context *context, uint8_t port_num, - struct ibv_port_attr *port_attr) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_query_port(context, port_num, port_attr); - if (sampling) SAMPLING_START(); - - return rc; -} -static -struct ibv_mr *ibv_reg_mr_wrapper(struct ibv_pd *pd, void *addr, - size_t length, int access) -{ - struct ibv_mr *mr=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - mr=ibv_reg_mr(pd, addr, length, access); - if (sampling) SAMPLING_START(); - - return mr; -} -static -int ibv_req_notify_cq_wrapper(struct ibv_cq *cq, int solicited_only) -{ - int rc=0; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - rc=ibv_req_notify_cq(cq, solicited_only); - if (sampling) SAMPLING_START(); - - return rc; -} -static -const char *ibv_wc_status_str_wrapper(enum ibv_wc_status status) -{ - const char *str=NULL; - - bool sampling = SAMPLING_IS_ACTIVE(); - if (sampling) SAMPLING_STOP(); - str=ibv_wc_status_str(status); - if (sampling) SAMPLING_START(); - - return str; -} - - - -/** - * @brief Initialize NNTI to use a specific transport. - * - * Enable the use of a particular transport by this process. my_url - * allows the process to have some control (if possible) over the - * URL assigned for the transport. For example, a Portals URL to put - * might be "ptl://-1,128". This would tell Portals to use the default - * network ID, but use PID=128. If the transport - * can be initialized without this info (eg. a Portals client), my_url can - * be NULL or empty. - */ -NNTI_result_t NNTI_ib_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl) -{ - int rc=0; - - struct ibv_device_attr dev_attr; - struct ibv_port_attr dev_port_attr; -// uint32_t cqe_count; -// uint32_t srq_count; - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; - char *sep; -// char *endptr; - - char hostname[NNTI_HOSTNAME_LEN]; -// NNTI_ip_addr addr; -// NNTI_tcp_port port; - - assert(trans_hdl); - - - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "my_url=%s", my_url); - log_debug(nnti_debug_level, "initialized=%d, FALSE==%d", (int)ib_initialized, (int)FALSE); - - if (!ib_initialized) { - - nthread_lock_init(&nnti_ib_lock); - - nthread_lock_init(&nnti_progress_lock); - nthread_cond_init(&nnti_progress_cond); - - nthread_lock_init(&nnti_conn_peer_lock); - nthread_lock_init(&nnti_conn_qpn_lock); - nthread_lock_init(&nnti_buf_bufhash_lock); - - nthread_lock_init(&nnti_wrmap_lock); - nthread_counter_init(&nnti_wrmap_counter); - - nthread_lock_init(&transport_global_data.atomics_lock); - - nthread_lock_init(&nnti_wr_pool_lock); - - config_init(&config); - config_get_from_env(&config); - - log_debug(nnti_debug_level, "my_url=%s", my_url); - - if (my_url != NULL) { - if ((rc=nnti_url_get_transport(my_url, transport, NNTI_URL_LEN)) != NNTI_OK) { - return(NNTI_EINVAL); - } - if (0!=strcmp(transport, "ib")) { - return(NNTI_EINVAL); - } - - if ((rc=nnti_url_get_address(my_url, address, NNTI_URL_LEN)) != NNTI_OK) { - return(NNTI_EINVAL); - } - - sep=strchr(address, ':'); - if (sep == address) { - /* no hostname given; try gethostname */ - gethostname(hostname, NNTI_HOSTNAME_LEN); - } else { - strncpy(hostname, address, sep-address); - hostname[sep-address]='\0'; - } -// sep++; -// port=strtol(sep, &endptr, 0); -// if (endptr == sep) { -// /* no port given; use -1 */ -// port=-1; -// } - } else { - gethostname(hostname, NNTI_HOSTNAME_LEN); -// port=-1; - } - strcpy(transport_global_data.listen_name, hostname); - - - log_debug(nnti_debug_level, "initializing InfiniBand"); - -// /* register trace groups (let someone else enable) */ -// trace_counter_gid = trace_get_gid(TRACE_RPC_COUNTER_GNAME); -// trace_interval_gid = trace_get_gid(TRACE_RPC_INTERVAL_GNAME); - - - memset(&transport_global_data, 0, sizeof(ib_transport_global)); - - struct ibv_device *dev=get_ib_device(); - - /* open the device */ - transport_global_data.ctx = ibv_open_device_wrapper(dev); - if (!transport_global_data.ctx) { - log_error(nnti_debug_level, "ibv_open_device failed"); - return NNTI_EIO; - } - - transport_global_data.nic_port = 1; - - /* get the lid and verify port state */ - rc = ibv_query_port_wrapper(transport_global_data.ctx, transport_global_data.nic_port, &dev_port_attr); - if (rc) { - log_error(nnti_debug_level, "ibv_query_port failed"); - return NNTI_EIO; - } - - transport_global_data.nic_lid = dev_port_attr.lid; - - if (dev_port_attr.state != IBV_PORT_ACTIVE) { - log_error(nnti_debug_level, "port is not active. cannot continue."); - return NNTI_EIO; - } - - /* Query the device for the max_ requests and such */ - rc = ibv_query_device_wrapper(transport_global_data.ctx, &dev_attr); - if (rc) { - log_error(nnti_debug_level, "ibv_query_device failed"); - return NNTI_EIO; - } - - log_debug(nnti_debug_level, "max %d completion queue entries", dev_attr.max_cqe); - transport_global_data.cqe_count = dev_attr.max_cqe; - - log_debug(nnti_debug_level, "max %d shared receive queue work requests", dev_attr.max_srq_wr); - transport_global_data.srq_count = dev_attr.max_srq_wr/2; - - log_debug(nnti_debug_level, "max %d shared receive queue scatter gather elements", dev_attr.max_srq_sge); - transport_global_data.sge_count = dev_attr.max_srq_sge; - - log_debug(nnti_debug_level, "max %d queue pair work requests", dev_attr.max_qp_wr); - transport_global_data.qp_count = dev_attr.max_qp_wr/2; - - /* Allocate a Protection Domain (global) */ - transport_global_data.pd = ibv_alloc_pd_wrapper(transport_global_data.ctx); - if (!transport_global_data.pd) { - log_error(nnti_debug_level, "ibv_alloc_pd failed"); - return NNTI_EIO; - } - - setup_request_channel(); - setup_data_channel(); - - if (config.use_wr_pool) { - rc=wr_pool_init(101); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "wr_pool_init(): %d", rc); - goto cleanup; - } - } - - setup_interrupt_pipe(); - - setup_atomics(); - - init_server_listen_socket(); - - if (logging_info(nnti_debug_level)) { - fprintf(logger_get_file(), "InfiniBand Initialized: host(%s) port(%u)\n", - transport_global_data.listen_name, - ntohs(transport_global_data.listen_port)); - } - - create_peer( - &transport_global_data.me, - transport_global_data.listen_name, - transport_global_data.listen_addr, - transport_global_data.listen_port); - copy_peer( - &trans_hdl->me, - &transport_global_data.me); - - ib_initialized = true; - } - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(NNTI_OK); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recognize. - * - * URL format: "transport://address/memory_descriptor" - * - transport - (required) identifies how the URL should parsed - * - address - (required) uniquely identifies a location on the network - * - ex. "ptl://nid:pid/", "ib://ip_addr:port" - * - memory_descriptor - (optional) transport-specific representation of RMA params - */ -NNTI_result_t NNTI_ib_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen) -{ - NNTI_result_t rc=NNTI_OK; - - assert(trans_hdl); - assert(url); - assert(maxlen>0); - - strncpy(url, trans_hdl->me.url, maxlen); - url[maxlen-1]='\0'; - - return(rc); -} - - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * - * - * Connectionless transport: parse and populate - * Connected transport: parse, connection and populate - * - */ -NNTI_result_t NNTI_ib_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - trios_declare_timer(callTime); - - NNTI_peer_t *existing_peer=NULL; - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; - char *sep; - - char hostname[NNTI_HOSTNAME_LEN]; - NNTI_tcp_port port; - int s; - struct hostent *host_entry; - struct sockaddr_in skin; - - ib_connection *conn=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(peer_hdl); - - existing_peer=get_peer_by_url(url); - if (existing_peer!=NULL) { - *peer_hdl=*existing_peer; - return(NNTI_OK); - } - - if (url != NULL) { - if ((rc=nnti_url_get_transport(url, transport, NNTI_URL_LEN)) != NNTI_OK) { - return(rc); - } - if (0!=strcmp(transport, "ib")) { - /* the peer described by 'url' is not an IB peer */ - return(NNTI_EINVAL); - } - - if ((rc=nnti_url_get_address(url, address, NNTI_URL_LEN)) != NNTI_OK) { - return(rc); - } - - sep=strchr(address, ':'); - strncpy(hostname, address, sep-address); - hostname[sep-address]='\0'; - port=strtol(sep+1, NULL, 0); - } else { - /* */ - return(NNTI_EINVAL); - } - - - - s = socket(AF_INET, SOCK_STREAM, 0); - if (s < 0) { - log_warn(nnti_debug_level, "failed to create tcp socket: errno=%d (%s)", errno, strerror(errno)); - return NNTI_EIO; - } - - host_entry = gethostbyname(hostname); - if (!host_entry) { - log_warn(nnti_debug_level, "failed to resolve server name (%s): %s", hostname, strerror(errno)); - return NNTI_ENOENT; - } - memset(&skin, 0, sizeof(skin)); - skin.sin_family = host_entry->h_addrtype; - memcpy(&skin.sin_addr, host_entry->h_addr_list[0], (size_t) host_entry->h_length); - skin.sin_port = htons(port); - - trios_start_timer(callTime); -retry: - if (connect(s, (struct sockaddr *) &skin, sizeof(skin)) < 0) { - if (errno == EINTR) { - goto retry; - } else { - log_warn(nnti_debug_level, "failed to connect to server (%s:%u): errno=%d (%s)", hostname, port, errno, strerror(errno)); - return NNTI_EIO; - } - } - trios_stop_timer("socket connect", callTime); - - conn = (ib_connection *)calloc(1, sizeof(ib_connection)); - log_debug(nnti_debug_level, "calloc returned conn=%p.", conn); - if (conn == NULL) { - log_error(nnti_debug_level, "calloc returned NULL. out of memory?: %s", strerror(errno)); - rc=NNTI_ENOMEM; - goto cleanup; - } - - trios_start_timer(callTime); - init_connection(&conn, s, 0); - trios_stop_timer("ib init connection", callTime); - - create_peer( - peer_hdl, - conn->peer_name, - conn->peer_addr, - conn->peer_port); - - conn->peer=*peer_hdl; - - insert_conn_qpn(conn->req_qp.qpn, conn); - insert_conn_qpn(conn->data_qp.qpn, conn); - insert_conn_peer(peer_hdl, conn); - - transition_connection_to_ready(s, conn); - - if (close(s) < 0) { - log_warn(nnti_debug_level, "failed to close tcp socket: errno=%d (%s)", errno, strerror(errno)); - return NNTI_EIO; - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer_hdl", - "end of NNTI_ib_connect", peer_hdl); - } - -cleanup: - log_debug(nnti_debug_level, "enter"); - - return(rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - */ -NNTI_result_t NNTI_ib_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "Disconnecting from IB"); - - assert(trans_hdl); - assert(peer_hdl); - - ib_connection *conn=get_conn_peer(peer_hdl); - close_connection(conn); - del_conn_peer(peer_hdl); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_ib_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - trios_declare_timer(callTime); - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - char *buf=(char *)aligned_malloc(element_size*num_elements); - assert(buf); - - nnti_rc=NNTI_ib_register_memory( - trans_hdl, - buf, - element_size, - num_elements, - ops, - reg_buf); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_ib_alloc", reg_buf); - } - - if (config.use_memset) { - trios_start_timer(callTime); - memset(buf, 0, element_size*num_elements); - trios_stop_timer("memset", callTime); - } - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_ib_free ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - char *buf=NNTI_BUFFER_C_POINTER(reg_buf); - assert(buf); - - nnti_rc=NNTI_ib_unregister_memory(reg_buf); - - free(buf); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_ib_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - uint32_t cqe_num; - - NNTI_buffer_t *old_buf=NULL; - ib_memory_handle *ib_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(buffer); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - old_buf=get_buf_bufhash(hash6432shift((uint64_t)buffer)); - if (old_buf==NULL) { - ib_mem_hdl=new ib_memory_handle(); - assert(ib_mem_hdl); - ib_mem_hdl->ref_count=1; - nthread_lock_init(&ib_mem_hdl->wr_queue_lock); - } else { - ib_mem_hdl=IB_MEM_HDL(old_buf); - ib_mem_hdl->ref_count++; - } - - log_debug(nnti_debug_level, "ib_mem_hdl->ref_count==%lu", ib_mem_hdl->ref_count); - - memset(reg_buf, 0, sizeof(NNTI_buffer_t)); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size = element_size; - reg_buf->payload = (uint64_t)buffer; - reg_buf->transport_private = (uint64_t)ib_mem_hdl; - - log_debug(nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - - if (ib_mem_hdl->ref_count==1) { - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(1, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=1; - - ib_mem_hdl->mr_list=&ib_mem_hdl->mr; - ib_mem_hdl->mr_count=1; - - if (ops == NNTI_BOP_RECV_QUEUE) { - ib_request_queue_handle *q_hdl=&transport_global_data.req_queue; - - q_hdl->reg_buf=reg_buf; - - q_hdl->req_buffer =buffer; - q_hdl->req_size =element_size; - q_hdl->req_count =num_elements; - q_hdl->req_received=0; - - reg_buf->payload_size=q_hdl->req_size; - - cqe_num=q_hdl->req_count; - if (cqe_num >= transport_global_data.srq_count) { - cqe_num = transport_global_data.srq_count; - } - - ib_mem_hdl->mr=register_memory_segment( - buffer, - num_elements*element_size, - IBV_ACCESS_LOCAL_WRITE); - - log_debug(nnti_debug_level, "mr=%p mr_list[0]=%p", ib_mem_hdl->mr, ib_mem_hdl->mr_list[0]); - - for (uint32_t i=0;ireq_size)); - } - - } else { - ib_mem_hdl->mr=register_memory_segment( - buffer, - element_size, - (ibv_access_flags)(IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE)); - - log_debug(nnti_debug_level, "mr=%p mr_list[0]=%p", ib_mem_hdl->mr, ib_mem_hdl->mr_list[0]); - - if ((ops & NNTI_BOP_REMOTE_WRITE) || - (ops & NNTI_BOP_REMOTE_READ)) { - post_recv_work_request( - reg_buf, - -1, - 0); - } - } - } else { - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(1, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=1; - } - - if (config.use_rdma_target_ack) { - if ((ops & NNTI_BOP_REMOTE_READ) || - (ops & NNTI_BOP_REMOTE_WRITE)) { - post_ack_recv_work_request(reg_buf); - } - } - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id = NNTI_TRANSPORT_IB; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.size = ib_mem_hdl->mr_list[0]->length; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf = (uint64_t)ib_mem_hdl->mr_list[0]->addr; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.key = ib_mem_hdl->mr_list[0]->rkey; - - if (ib_mem_hdl->ref_count==1) { - insert_buf_bufhash(reg_buf); - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_ib_register_memory", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_ib_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - - uint32_t cqe_num; - - NNTI_buffer_t *old_buf=NULL; - ib_memory_handle *ib_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(segments); - assert(segment_lengths>0); - assert(num_segments>0); - assert(ops>0); - assert(reg_buf); - - if (ops == NNTI_BOP_RECV_QUEUE) { - log_debug(nnti_debug_level, "NNTI_BOP_RECV_QUEUE cannot be segmented."); - return(NNTI_EINVAL); - } - - old_buf=get_buf_bufhash(hash6432shift((uint64_t)segments[0])); - if (old_buf==NULL) { - ib_mem_hdl=new ib_memory_handle(); - assert(ib_mem_hdl); - ib_mem_hdl->ref_count=1; - nthread_lock_init(&ib_mem_hdl->wr_queue_lock); - } else { - // check that the number of old_buf segments equals num_segments. - if (old_buf->buffer_segments.NNTI_remote_addr_array_t_len != num_segments) { - log_debug(nnti_debug_level, "Segment count mismatch (old=%d new=%d). Aborting...", - old_buf->buffer_segments.NNTI_remote_addr_array_t_len, num_segments); - return(NNTI_EINVAL); - } - // check that all the old_buf segments match the current list of segments - for (int i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.buf != (uint64_t)segments[i]) { - log_debug(nnti_debug_level, "Segment address mismatch (old[%d]=%p new[%d]=%p). Aborting...", - i, old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.buf, i, segments[i]); - return(NNTI_EINVAL); - } - if (old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size != segment_lengths[i]) { - log_debug(nnti_debug_level, "Segment length mismatch (old[%d]=%d new[%d]=%d). Aborting...", - i, old_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size, i, segment_lengths[i]); - return(NNTI_EINVAL); - } - } - ib_mem_hdl=IB_MEM_HDL(old_buf); - ib_mem_hdl->ref_count++; - } - - log_debug(nnti_debug_level, "ib_mem_hdl->ref_count==%lu", ib_mem_hdl->ref_count); - - memset(reg_buf, 0, sizeof(NNTI_buffer_t)); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size=0; - for (int i=0;ipayload_size += segment_lengths[i]; - } - reg_buf->payload = (uint64_t)segments[0]; - reg_buf->transport_private = (uint64_t)ib_mem_hdl; - - log_debug(nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - - if (ib_mem_hdl->ref_count==1) { - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(num_segments, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=num_segments; - - ib_mem_hdl->mr_list=(struct ibv_mr **)calloc(num_segments, sizeof(struct ibv_mr *)); - ib_mem_hdl->mr_count=num_segments; - - for (int i=0;imr_list[i]=register_memory_segment( - segments[i], - segment_lengths[i], - (ibv_access_flags)(IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE)); - - log_debug(nnti_debug_level, "mr_list[%d]=%p", i, ib_mem_hdl->mr_list[i]); - } - } - - if (config.use_rdma_target_ack) { - if ((ops & NNTI_BOP_REMOTE_READ) || - (ops & NNTI_BOP_REMOTE_WRITE)) { - post_ack_recv_work_request(reg_buf); - } - } - - for (int i=0;ibuffer_segments.NNTI_remote_addr_array_t_val[i].transport_id = NNTI_TRANSPORT_IB; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size = ib_mem_hdl->mr_list[i]->length; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.buf = (uint64_t)ib_mem_hdl->mr_list[i]->addr; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.key = ib_mem_hdl->mr_list[i]->rkey; - } - - if (ib_mem_hdl->ref_count==1) { - insert_buf_bufhash(reg_buf); - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_ib_register_memory", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_ib_unregister_memory ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t rc=NNTI_OK; - ib_memory_handle *ib_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "start of NNTI_ib_unregister_memory", reg_buf); - } - - ib_mem_hdl=IB_MEM_HDL(reg_buf); - assert(ib_mem_hdl); - ib_mem_hdl->ref_count--; - - log_debug(nnti_debug_level, "ib_mem_hdl->ref_count==%lu", ib_mem_hdl->ref_count); - - if (ib_mem_hdl->ref_count==0) { - log_debug(nnti_debug_level, "ib_mem_hdl->ref_count is 0. release all resources."); - log_debug(nnti_debug_level, "This buffer has %d segments.", ib_mem_hdl->mr_count); - for (int i=0;imr_count;i++) { - log_debug(nnti_debug_level, "Unregistering segment #%d.", i); - unregister_memory_segment(ib_mem_hdl->mr_list[i]); - } - - del_buf_bufhash(reg_buf); - - nthread_lock(&ib_mem_hdl->wr_queue_lock); - while (!ib_mem_hdl->wr_queue.empty()) { - ib_work_request *ib_wr=ib_mem_hdl->wr_queue.front(); - log_debug(nnti_debug_level, "popping pending (reg_buf=%p, ib_wr=%p)", reg_buf, ib_wr); - if (ib_wr==NULL) - break; - ib_mem_hdl->wr_queue.pop_front(); - if (config.use_wr_pool) { - if (ib_wr->ack_mr!=NULL) { - wr_pool_rdma_push(ib_wr); - } else { - wr_pool_sendrecv_push(ib_wr); - } - } else { - if (config.use_rdma_target_ack) { - unregister_ack(ib_wr); - } - log_debug(nnti_debug_level, "freeing ib_wr=%p", ib_wr); - free(ib_wr); - } - } - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - nthread_lock_fini(&ib_mem_hdl->wr_queue_lock); - - if (reg_buf->buffer_segments.NNTI_remote_addr_array_t_val != NULL) { - free(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val); - } - if ((ib_mem_hdl->mr_count > 1) && (ib_mem_hdl->mr_list != NULL)) { - free(ib_mem_hdl->mr_list); - } - - if (ib_mem_hdl) delete ib_mem_hdl; - - reg_buf->transport_id = NNTI_TRANSPORT_NULL; - IB_SET_MATCH_ANY(®_buf->buffer_owner); - reg_buf->ops = (NNTI_buf_ops_t)0; - reg_buf->payload_size = 0; - reg_buf->payload = 0; - reg_buf->transport_private = 0; - } - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - */ -NNTI_result_t NNTI_ib_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - trios_declare_timer(call_time); - - struct ibv_send_wr *bad_wr; - - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - assert(msg_hdl); - - log_level debug_level=nnti_debug_level; - - ib_mem_hdl=IB_MEM_HDL(msg_hdl); - assert(ib_mem_hdl); - if (config.use_wr_pool) { - ib_wr=wr_pool_sendrecv_pop(); - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - log_debug(nnti_debug_level, "allocated ib_wr (wr=%p ; ib_wr=%p)", wr, ib_wr); - nthread_lock_init(&ib_wr->lock); - } - assert(ib_wr); - - ib_wr->conn = get_conn_peer(peer_hdl); - assert(ib_wr->conn); - - ib_wr->nnti_wr = wr; - ib_wr->reg_buf = (NNTI_buffer_t *)msg_hdl; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - ib_wr->sq_wr_list =&ib_wr->sq_wr; - ib_wr->sq_wr_count=1; - - if ((dest_hdl == NULL) || (dest_hdl->ops == NNTI_BOP_RECV_QUEUE)) { - ib_wr->comp_channel=transport_global_data.req_comp_channel; - ib_wr->cq =transport_global_data.req_cq; - ib_wr->qp =ib_wr->conn->req_qp.qp; - ib_wr->qpn =(uint64_t)ib_wr->conn->req_qp.qpn; - ib_wr->peer_qpn =(uint64_t)ib_wr->conn->peer_req_qpn; - - ib_wr->sq_wr.opcode =IBV_WR_SEND; - ib_wr->sq_wr.send_flags=IBV_SEND_SIGNALED; - - ib_wr->last_op=IB_OP_SEND_REQUEST; - } else { - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - ib_wr->qp =ib_wr->conn->data_qp.qp; - ib_wr->qpn =(uint64_t)ib_wr->conn->data_qp.qpn; - ib_wr->peer_qpn =(uint64_t)ib_wr->conn->data_qp.peer_qpn; - - ib_wr->sq_wr.wr.rdma.rkey =dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.key; - ib_wr->sq_wr.wr.rdma.remote_addr=dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf; - - ib_wr->sq_wr.imm_data =hash6432shift((uint64_t)dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - ib_wr->sq_wr.opcode =IBV_WR_RDMA_WRITE_WITH_IMM; - ib_wr->sq_wr.send_flags=IBV_SEND_SIGNALED; - - ib_wr->last_op=IB_OP_SEND_BUFFER; - } - - // setup the scatter-gather list for this work request - ib_wr->sge_count=msg_hdl->buffer_segments.NNTI_remote_addr_array_t_len; - if (ib_wr->sge_count == 1) { - ib_wr->sge_list=&ib_wr->sge; - ib_wr->sge_list[0].addr =(uint64_t)ib_mem_hdl->mr_list[0]->addr; - ib_wr->sge_list[0].length=msg_hdl->payload_size; - ib_wr->sge_list[0].lkey =ib_mem_hdl->mr_list[0]->lkey; - } else { - ib_wr->sge_list=(struct ibv_sge *)calloc(ib_wr->sge_count, sizeof(struct ibv_sge)); - for (int i=0;isge_count;i++) { - ib_wr->sge_list[i].addr =(uint64_t)ib_mem_hdl->mr_list[i]->addr; - ib_wr->sge_list[i].length=ib_mem_hdl->mr_list[i]->length; - ib_wr->sge_list[i].lkey =ib_mem_hdl->mr_list[i]->lkey; - } - } - ib_wr->sq_wr.wr_id = (uint64_t)ib_wr->key; - ib_wr->sq_wr.next = NULL; // RAOLDFI ADDED - ib_wr->sq_wr.sg_list = ib_wr->sge_list; - ib_wr->sq_wr.num_sge = ib_wr->sge_count; - - - log_debug(nnti_debug_level, "sending to (%s, qp=%p, qpn=%d, sge.addr=%p, sge.length=%llu, sq_wr.imm_data=0x%llx, sq_wr.ib_wr.rdma.rkey=0x%x, sq_wr.ib_wr.rdma.remote_addr=%p)", - peer_hdl->url, - ib_wr->qp, - ib_wr->qpn, - (void *) ib_wr->sge.addr, - (uint64_t)ib_wr->sge.length, - (uint64_t)ib_wr->sq_wr.imm_data, - ib_wr->sq_wr.wr.rdma.rkey, - (void *) ib_wr->sq_wr.wr.rdma.remote_addr); - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - nthread_lock(&ib_mem_hdl->wr_queue_lock); - ib_mem_hdl->wr_queue.push_back(ib_wr); - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - wr->transport_id =msg_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)msg_hdl; - wr->ops =NNTI_BOP_LOCAL_READ; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)ib_wr; - - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - trios_start_timer(call_time); - if (ibv_post_send_wrapper(ib_wr->qp, &ib_wr->sq_wr, &bad_wr)) { - log_error(nnti_debug_level, "failed to post send: %s", strerror(errno)); - rc=NNTI_EIO; - } - trios_stop_timer("NNTI_ib_send - ibv_post_send", call_time); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_ib_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - NNTI_remote_addr_t *dst_remote_addr; - NNTI_remote_addr_t *src_remote_addr; - - trios_declare_timer(call_time); - - struct ibv_send_wr *bad_wr=NULL; - - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - uint64_t remaining=0; - - struct { - uint32_t segment_index; // index of the current segment - uint64_t segment_offset; // offset into the current segment - uint64_t segment_remaining; // bytes remaining in the current segment - uint32_t last_segment; // index of the last segment in this transfer - } src_params, dst_params, src_copy, dst_copy; - - uint32_t wr_index=0; - - - log_debug(nnti_debug_level, "enter"); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - ib_mem_hdl=IB_MEM_HDL(src_buffer_hdl); - assert(ib_mem_hdl); - if (config.use_wr_pool) { - if (config.use_rdma_target_ack) { - ib_wr=wr_pool_rdma_pop(); - } else { - ib_wr=wr_pool_sendrecv_pop(); - } - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - log_debug(nnti_debug_level, "allocated ib_wr (wr=%p ; ib_wr=%p)", wr, ib_wr); - nthread_lock_init(&ib_wr->lock); - } - assert(ib_wr); - - ib_wr->conn = get_conn_peer(&dest_buffer_hdl->buffer_owner); - assert(ib_wr->conn); - - ib_wr->nnti_wr = wr; - ib_wr->reg_buf = (NNTI_buffer_t *)src_buffer_hdl; - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - ib_wr->qp =ib_wr->conn->data_qp.qp; - ib_wr->qpn =(uint64_t)ib_wr->conn->data_qp.qpn; - ib_wr->peer_qpn =(uint64_t)ib_wr->conn->data_qp.peer_qpn; - - ib_wr->sq_wr_completed_count=0; - - if (dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_len == 1) { - // this is the easy case. the destination (remote) buffer is contiguous so we can complete this PUT with one ibv_send_wr. - - ib_wr->sq_wr_list =&ib_wr->sq_wr; - ib_wr->sq_wr_count=1; - - // the src_offset could exclude some local segments from this operation. find the first segment. - uint64_t segment_offset=src_offset; - uint32_t first_segment =0; - for (int i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (segment_offset > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - segment_offset -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - continue; - } else if (segment_offset == src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - segment_offset = 0; - first_segment = i+1; - break; - } else { - first_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "first_segment=%u, segment_offset=%lu", first_segment, segment_offset); - - // setup the scatter-gather list for this work request - ib_wr->sge_count=src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_len - first_segment; - if (ib_wr->sge_count == 1) { - ib_wr->sge_list=&ib_wr->sge; - ib_wr->sge_list[0].addr =(uint64_t)ib_mem_hdl->mr_list[first_segment]->addr + segment_offset; - ib_wr->sge_list[0].length=src_length; - ib_wr->sge_list[0].lkey =ib_mem_hdl->mr_list[first_segment]->lkey; - } else { - ib_wr->sge_list=(struct ibv_sge *)calloc(ib_wr->sge_count, sizeof(struct ibv_sge)); - for (int i=0,j=first_segment;isge_count;i++,j++) { - ib_wr->sge_list[i].addr=(uint64_t)ib_mem_hdl->mr_list[j]->addr + segment_offset; - if (i == 0) { - ib_wr->sge_list[i].length=ib_mem_hdl->mr_list[j]->length - segment_offset; - } else { - ib_wr->sge_list[i].length=ib_mem_hdl->mr_list[j]->length; - } - ib_wr->sge_list[i].lkey=ib_mem_hdl->mr_list[j]->lkey; - } - } - ib_wr->sq_wr.sg_list=ib_wr->sge_list; - ib_wr->sq_wr.num_sge=ib_wr->sge_count; - - ib_wr->sq_wr.wr.rdma.rkey = dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.key; - ib_wr->sq_wr.wr.rdma.remote_addr = dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf+dest_offset; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->sq_wr.opcode =IBV_WR_RDMA_WRITE; - ib_wr->sq_wr.send_flags=IBV_SEND_SIGNALED; - ib_wr->sq_wr.wr_id =(uint64_t)ib_wr->key; - ib_wr->sq_wr.imm_data =hash6432shift((uint64_t)dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - ib_wr->sq_wr.next =NULL; // RAOLDFI ADDED - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - } else { - // the src_offset could exclude some local segments from this operation. find the first segment. - src_params.segment_offset =src_offset; - src_params.segment_remaining=0; - src_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "src_offset=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_buffer.segment[%llu].size==%llu", - src_offset, src_params.segment_offset, src_params.segment_remaining, - (uint64_t)i, (uint64_t)src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size); - if (src_params.segment_offset > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - src_params.segment_offset -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - - log_debug(nnti_debug_level, "src_params.segment_index=%lu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - continue; - } else { - src_params.segment_index = i; - src_params.segment_remaining = src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size - src_params.segment_offset; - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=src_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - remaining -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - } else { - src_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_params.last_segment=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining, (uint64_t)src_params.last_segment); - - // the dest_offset could exclude some remote segments from this operation. find the first segment. - dst_params.segment_offset =dest_offset; - dst_params.segment_remaining=0; - dst_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "dest_offset=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dest_buffer.segment[%llu].size==%llu", - dest_offset, dst_params.segment_offset, dst_params.segment_remaining, - (uint64_t)i, (uint64_t)dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size); - if (dst_params.segment_offset > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - dst_params.segment_offset -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - continue; - } else { - dst_params.segment_index = i; - dst_params.segment_remaining = dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size - dst_params.segment_offset; - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=dst_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - remaining -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - } else { - dst_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dst_params.last_segment=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining, (uint64_t)dst_params.last_segment); - - /* START calculate the number of SGEs required */ - { - src_copy=src_params; - dst_copy=dst_params; - - src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_copy.segment_index]; - dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_copy.segment_index]; - - ib_wr->sq_wr_count=0; - - while (src_copy.segment_index <= src_copy.last_segment) { - - ib_wr->sq_wr_count++; - - if (src_copy.segment_remaining < dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment with extra space in the destination"); - // the remaining source segment fits in the remaining destination segment with extra space - dst_copy.segment_offset += src_copy.segment_remaining; - dst_copy.segment_remaining -= src_copy.segment_remaining; - - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else if (src_copy.segment_remaining == dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment EXACTLY"); - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else { - log_debug(nnti_debug_level, "the remaining source segment DOES NOT fit in the remaining destination segment"); - // the remaining source segment DOES NOT fit in the remaining destination segment - src_copy.segment_offset += dst_copy.segment_remaining; - src_copy.segment_remaining -= dst_copy.segment_remaining; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } - - log_debug(nnti_debug_level, "src_copy.segment_index=%llu, src_copy.segment_offset=%llu, src_copy.segment_remaining=%llu, src_copy.last_segment=%llu", - (uint64_t)src_copy.segment_index, src_copy.segment_offset, (uint64_t)src_copy.segment_remaining, (uint64_t)src_copy.last_segment); - log_debug(nnti_debug_level, "dst_copy.segment_index=%llu, dst_copy.segment_offset=%llu, dst_copy.segment_remaining=%llu, dst_copy.last_segment=%llu", - (uint64_t)dst_copy.segment_index, dst_copy.segment_offset, (uint64_t)dst_copy.segment_remaining, (uint64_t)dst_copy.last_segment); - } - } - /* END calculate the number of SGEs required */ - - src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_params.segment_index]; - dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_params.segment_index]; - - ib_wr->sq_wr_list =(struct ibv_send_wr *)malloc(ib_wr->sq_wr_count*sizeof(struct ibv_send_wr)); - - ib_wr->sge_count =ib_wr->sq_wr_count; - ib_wr->sge_list =(struct ibv_sge *)malloc(ib_wr->sge_count*sizeof(struct ibv_sge)); - - log_debug(nnti_debug_level, "this get requires %d WRs", ib_wr->sq_wr_count); - - wr_index=0; - - while (src_params.segment_index <= src_params.last_segment) { - - ib_wr->sq_wr_list[wr_index].sg_list = &ib_wr->sge_list[wr_index]; - ib_wr->sq_wr_list[wr_index].num_sge = 1; - - ib_wr->sq_wr_list[wr_index].sg_list[0].addr=(uint64_t)ib_mem_hdl->mr_list[src_params.segment_index]->addr + src_params.segment_offset; - ib_wr->sq_wr_list[wr_index].sg_list[0].lkey=ib_mem_hdl->mr_list[src_params.segment_index]->lkey; - - if (src_params.segment_remaining <= dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - ib_wr->sq_wr_list[wr_index].sg_list[0].length=src_params.segment_remaining; - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - ib_wr->sq_wr_list[wr_index].sg_list[0].length=dst_params.segment_remaining; - } - - ib_wr->sq_wr_list[wr_index].wr.rdma.rkey = dst_remote_addr->NNTI_remote_addr_t_u.ib.key; - ib_wr->sq_wr_list[wr_index].wr.rdma.remote_addr = dst_remote_addr->NNTI_remote_addr_t_u.ib.buf+dst_params.segment_offset; - - ib_wr->sq_wr_list[wr_index].opcode =IBV_WR_RDMA_WRITE; - ib_wr->sq_wr_list[wr_index].send_flags=IBV_SEND_SIGNALED; - ib_wr->sq_wr_list[wr_index].wr_id =(uint64_t)nthread_counter_increment(&nnti_wrmap_counter); - ib_wr->sq_wr_list[wr_index].imm_data =hash6432shift((uint64_t)dst_remote_addr->NNTI_remote_addr_t_u.ib.buf); - ib_wr->sq_wr_list[wr_index].next =NULL; // RAOLDFI ADDED - - log_debug(nnti_debug_level, "ib_wr->sq_wr_list[%d].sg_list[0].addr=0x%lX ; ib_wr->sq_wr_list[%d].sg_list[0].length=%u ; ib_wr->sq_wr_list[%d].sg_list[0].lkey=0x%X", - wr_index, ib_wr->sq_wr_list[wr_index].sg_list[0].addr, wr_index, ib_wr->sq_wr_list[wr_index].sg_list[0].length, wr_index, ib_wr->sq_wr_list[wr_index].sg_list[0].lkey); - log_debug(nnti_debug_level, "ib_wr->sq_wr_list[%d].wr.rdma._remote_addr=0x%lX ; ib_wr->sq_wr_list[%d].wr.rdma.rkey=0x%X", - wr_index, ib_wr->sq_wr_list[wr_index].wr.rdma.remote_addr, wr_index, ib_wr->sq_wr_list[wr_index].wr.rdma.rkey); - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->sq_wr_list[wr_index].wr_id, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->sq_wr_list[wr_index].wr_id) == wrmap.end()); - wrmap[ib_wr->sq_wr_list[wr_index].wr_id] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - if (src_params.segment_remaining < dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - dst_params.segment_offset += src_params.segment_remaining; - dst_params.segment_remaining -= src_params.segment_remaining; - - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else if (src_params.segment_remaining == dst_params.segment_remaining) { - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - src_params.segment_offset += dst_params.segment_remaining; - src_params.segment_remaining -= dst_params.segment_remaining; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } - - wr_index++; - } - } - - if (config.use_rdma_target_ack) { - ib_wr->ack.op =IB_OP_PUT_TARGET; - ib_wr->ack.offset=dest_offset; - ib_wr->ack.length=src_length; - - if (!config.use_wr_pool) { - register_ack(ib_wr); - } - ib_wr->ack_sge.addr =(uint64_t)ib_wr->ack_mr->addr; - ib_wr->ack_sge.length=ib_wr->ack_mr->length; - ib_wr->ack_sge.lkey =ib_wr->ack_mr->lkey; - - ib_wr->ack_sq_wr.sg_list=&ib_wr->ack_sge; - ib_wr->ack_sq_wr.num_sge=1; - - ib_wr->ack_sq_wr.wr.rdma.rkey =dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_key; - ib_wr->ack_sq_wr.wr.rdma.remote_addr=dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_buf; - - ib_wr->ack_sq_wr.opcode =IBV_WR_RDMA_WRITE_WITH_IMM; - ib_wr->ack_sq_wr.send_flags=IBV_SEND_SIGNALED|IBV_SEND_FENCE; - ib_wr->sq_wr.wr_id =(uint64_t)ib_wr->key; - ib_wr->ack_sq_wr.imm_data =hash6432shift((uint64_t)dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - ib_wr->ack_sq_wr.next =NULL; // RAOLDFI ADDED - } - - ib_wr->last_op=IB_OP_PUT_INITIATOR; - ib_wr->length=src_length; - ib_wr->offset=src_offset; - - log_debug(nnti_debug_level, "putting to (%s, qp=%p, qpn=%lu)", - dest_buffer_hdl->buffer_owner.url, - ib_wr->qp, - ib_wr->qpn); - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - nthread_lock(&ib_mem_hdl->wr_queue_lock); - ib_mem_hdl->wr_queue.push_back(ib_wr); - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - wr->transport_id =src_buffer_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)src_buffer_hdl; - wr->ops =NNTI_BOP_LOCAL_READ; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)ib_wr; - - for (int i=0;isq_wr_count;i++) { - log_debug(nnti_debug_level, "posting ib_wr->sq_wr_list[%d]=%p (key=%lx)", i, ib_wr->sq_wr_list[i], ib_wr->sq_wr_list[i].wr_id); - trios_start_timer(call_time); - if (ibv_post_send_wrapper(ib_wr->qp, &ib_wr->sq_wr_list[i], &bad_wr)) { - log_error(nnti_debug_level, "failed to post send: %s", strerror(errno)); - rc=NNTI_EIO; - } - trios_stop_timer("NNTI_ib_put - ibv_post_send", call_time); - - if (config.use_rdma_target_ack) { - send_ack(ib_wr); - } - - // print_wr(ib_wr); - } - - if (ib_wr->sge_list != &ib_wr->sge) { - free(ib_wr->sge_list); - } - if (ib_wr->sq_wr_list != &ib_wr->sq_wr) { - free(ib_wr->sq_wr_list); - } - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_ib_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - log_level debug_level = nnti_debug_level; - - NNTI_remote_addr_t *dst_remote_addr; - NNTI_remote_addr_t *src_remote_addr; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - struct ibv_send_wr *bad_wr=NULL; - - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - uint64_t remaining=0; - - struct { - uint32_t segment_index; // index of the current segment - uint64_t segment_offset; // offset into the current segment - uint64_t segment_remaining; // bytes remaining in the current segment - uint32_t last_segment; // index of the last segment in this transfer - } src_params, dst_params, src_copy, dst_copy; - - uint32_t wr_index=0; - - trios_start_timer(total_time); - - log_debug(debug_level, "enter (wr=%p)", wr); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - ib_mem_hdl=IB_MEM_HDL(dest_buffer_hdl); - assert(ib_mem_hdl); - if (config.use_wr_pool) { - if (config.use_rdma_target_ack) { - ib_wr=wr_pool_rdma_pop(); - } else { - ib_wr=wr_pool_sendrecv_pop(); - } - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - log_debug(nnti_debug_level, "allocated ib_wr (wr=%p ; ib_wr=%p)", wr, ib_wr); - nthread_lock_init(&ib_wr->lock); - } - assert(ib_wr); - - ib_wr->conn = get_conn_peer(&src_buffer_hdl->buffer_owner); - assert(ib_wr->conn); - - ib_wr->nnti_wr = wr; - ib_wr->reg_buf = (NNTI_buffer_t *)dest_buffer_hdl; - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - ib_wr->qp =ib_wr->conn->data_qp.qp; - ib_wr->qpn =(uint64_t)ib_wr->conn->data_qp.qpn; - ib_wr->peer_qpn =(uint64_t)ib_wr->conn->data_qp.peer_qpn; - - ib_wr->sq_wr_completed_count=0; - - if (src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_len == 1) { - // this is the easy case. the source (remote) buffer is contiguous so we can complete this GET with one ibv_send_wr. - - ib_wr->sq_wr_list =&ib_wr->sq_wr; - ib_wr->sq_wr_count=1; - - // the src_offset could exclude some local segments from this operation. find the first segment. - uint64_t segment_offset=dest_offset; - uint32_t first_segment =0; - for (int i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (segment_offset > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - segment_offset -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - continue; - } else if (segment_offset == dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - segment_offset = 0; - first_segment = i+1; - break; - } else { - first_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "first_segment=%u, segment_offset=%lu", first_segment, segment_offset); - - // setup the scatter-gather list for this work request - ib_wr->sge_count=dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_len - first_segment; - if (ib_wr->sge_count == 1) { - ib_wr->sge_list=&ib_wr->sge; - ib_wr->sge_list[0].addr =(uint64_t)ib_mem_hdl->mr_list[first_segment]->addr + segment_offset; - ib_wr->sge_list[0].length=src_length; - ib_wr->sge_list[0].lkey =ib_mem_hdl->mr_list[first_segment]->lkey; - } else { - ib_wr->sge_list=(struct ibv_sge *)calloc(ib_wr->sge_count, sizeof(struct ibv_sge)); - for (int i=0,j=first_segment;isge_count;i++,j++) { - ib_wr->sge_list[i].addr=(uint64_t)ib_mem_hdl->mr_list[j]->addr + segment_offset; - if (i == 0) { - ib_wr->sge_list[i].length=ib_mem_hdl->mr_list[j]->length - segment_offset; - } else { - ib_wr->sge_list[i].length=ib_mem_hdl->mr_list[j]->length; - } - ib_wr->sge_list[i].lkey=ib_mem_hdl->mr_list[j]->lkey; - } - } - ib_wr->sq_wr.sg_list=ib_wr->sge_list; - ib_wr->sq_wr.num_sge=ib_wr->sge_count; - - for (int i=0;isge_count;i++) { - log_debug(nnti_debug_level, "ib_wr->sge_list[%d].addr=0x%lX ; ib_wr->sge_list[%d].length=%u ; ib_wr->sge_list[%d].lkey=0x%X", - i, ib_wr->sge_list[i].addr, i, ib_wr->sge_list[i].length, i, ib_wr->sge_list[i].lkey); - } - for (int i=0;isq_wr.num_sge;i++) { - log_debug(nnti_debug_level, "ib_wr->sq_wr.sg_list[%d].addr=0x%lX ; ib_wr->sq_wr.sg_list[%d].length=%u ; ib_wr->sq_wr.sg_list[%d].lkey=0x%X", - i, ib_wr->sq_wr.sg_list[i].addr, i, ib_wr->sq_wr.sg_list[i].length, i, ib_wr->sq_wr.sg_list[i].lkey); - } - - - ib_wr->sq_wr.wr.rdma.rkey = src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.key; - ib_wr->sq_wr.wr.rdma.remote_addr = src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf+src_offset; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->sq_wr.opcode =IBV_WR_RDMA_READ; - ib_wr->sq_wr.send_flags=IBV_SEND_SIGNALED; - ib_wr->sq_wr.wr_id =(uint64_t)ib_wr->key; - ib_wr->sq_wr.imm_data =hash6432shift((uint64_t)src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - ib_wr->sq_wr.next =NULL; // RAOLDFI ADDED - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - } else { - // the dest_offset could exclude some local segments from this operation. find the first segment. - dst_params.segment_offset =dest_offset; - dst_params.segment_remaining=0; - dst_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "dest_offset=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dest_buffer.segment[%llu].size==%llu", - dest_offset, dst_params.segment_offset, dst_params.segment_remaining, - (uint64_t)i, (uint64_t)dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size); - if (dst_params.segment_offset > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - dst_params.segment_offset -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - - log_debug(nnti_debug_level, "dst_params.segment_index=%lu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - continue; - } else { - dst_params.segment_index = i; - dst_params.segment_remaining = dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size - dst_params.segment_offset; - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=dst_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - remaining -= dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - } else { - dst_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "dst_params.segment_index=%llu, dst_params.segment_offset=%llu, dst_params.segment_remaining=%llu, dst_params.last_segment=%llu", - (uint64_t)dst_params.segment_index, dst_params.segment_offset, dst_params.segment_remaining, (uint64_t)dst_params.last_segment); - - // the src_offset could exclude some remote segments from this operation. find the first segment. - src_params.segment_offset =src_offset; - src_params.segment_remaining=0; - src_params.segment_index =0; - for (uint32_t i=0;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - log_debug(nnti_debug_level, "src_offset=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_buffer.segment[%llu].size==%llu", - src_offset, src_params.segment_offset, src_params.segment_remaining, - (uint64_t)i, (uint64_t)src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size); - if (src_params.segment_offset > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - src_params.segment_offset -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - continue; - } else { - src_params.segment_index = i; - src_params.segment_remaining = src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size - src_params.segment_offset; - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining); - - break; - } - } - remaining=src_length; - for (uint32_t i=src_params.segment_index;ibuffer_segments.NNTI_remote_addr_array_t_len;i++) { - if (remaining > src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size) { - remaining -= src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[i].NNTI_remote_addr_t_u.ib.size; - } else { - src_params.last_segment = i; - break; - } - } - - log_debug(nnti_debug_level, "src_params.segment_index=%llu, src_params.segment_offset=%llu, src_params.segment_remaining=%llu, src_params.last_segment=%llu", - (uint64_t)src_params.segment_index, src_params.segment_offset, src_params.segment_remaining, (uint64_t)src_params.last_segment); - - /* START calculate the number of SGEs required */ - { - dst_copy=dst_params; - src_copy=src_params; - - dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_copy.segment_index]; - src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_copy.segment_index]; - - ib_wr->sq_wr_count=0; - - while (src_copy.segment_index <= src_copy.last_segment) { - - ib_wr->sq_wr_count++; - - if (src_copy.segment_remaining < dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment with extra space in the destination"); - // the remaining source segment fits in the remaining destination segment with extra space - dst_copy.segment_offset += src_copy.segment_remaining; - dst_copy.segment_remaining -= src_copy.segment_remaining; - - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else if (src_copy.segment_remaining == dst_copy.segment_remaining) { - log_debug(nnti_debug_level, "the remaining source segment fits in the remaining destination segment EXACTLY"); - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_copy.segment_index++; - src_copy.segment_offset=0; - src_copy.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else { - log_debug(nnti_debug_level, "the remaining source segment DOES NOT fit in the remaining destination segment"); - // the remaining source segment DOES NOT fit in the remaining destination segment - src_copy.segment_offset += dst_copy.segment_remaining; - src_copy.segment_remaining -= dst_copy.segment_remaining; - - dst_remote_addr++; - dst_copy.segment_index++; - dst_copy.segment_offset=0; - dst_copy.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } - - log_debug(nnti_debug_level, "src_copy.segment_index=%llu, src_copy.segment_offset=%llu, src_copy.segment_remaining=%llu, src_copy.last_segment=%llu", - (uint64_t)src_copy.segment_index, src_copy.segment_offset, (uint64_t)src_copy.segment_remaining, (uint64_t)src_copy.last_segment); - log_debug(nnti_debug_level, "dst_copy.segment_index=%llu, dst_copy.segment_offset=%llu, dst_copy.segment_remaining=%llu, dst_copy.last_segment=%llu", - (uint64_t)dst_copy.segment_index, dst_copy.segment_offset, (uint64_t)dst_copy.segment_remaining, (uint64_t)dst_copy.last_segment); - } - } - /* END calculate the number of SGEs required */ - - dst_remote_addr=&dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[dst_params.segment_index]; - src_remote_addr=&src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[src_params.segment_index]; - - ib_wr->sq_wr_list =(struct ibv_send_wr *)malloc(ib_wr->sq_wr_count*sizeof(struct ibv_send_wr)); - - ib_wr->sge_count =ib_wr->sq_wr_count; - ib_wr->sge_list =(struct ibv_sge *)malloc(ib_wr->sge_count*sizeof(struct ibv_sge)); - - log_debug(nnti_debug_level, "this get requires %d WRs", ib_wr->sq_wr_count); - - wr_index=0; - - while (src_params.segment_index <= src_params.last_segment) { - - ib_wr->sq_wr_list[wr_index].sg_list = &ib_wr->sge_list[wr_index]; - ib_wr->sq_wr_list[wr_index].num_sge = 1; - - ib_wr->sq_wr_list[wr_index].sg_list[0].addr=(uint64_t)ib_mem_hdl->mr_list[dst_params.segment_index]->addr + dst_params.segment_offset; - ib_wr->sq_wr_list[wr_index].sg_list[0].lkey=ib_mem_hdl->mr_list[dst_params.segment_index]->lkey; - - if (src_params.segment_remaining <= dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - ib_wr->sq_wr_list[wr_index].sg_list[0].length=src_params.segment_remaining; - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - ib_wr->sq_wr_list[wr_index].sg_list[0].length=dst_params.segment_remaining; - } - - ib_wr->sq_wr_list[wr_index].wr.rdma.rkey = src_remote_addr->NNTI_remote_addr_t_u.ib.key; - ib_wr->sq_wr_list[wr_index].wr.rdma.remote_addr = src_remote_addr->NNTI_remote_addr_t_u.ib.buf+src_params.segment_offset; - - ib_wr->sq_wr_list[wr_index].opcode =IBV_WR_RDMA_READ; - ib_wr->sq_wr_list[wr_index].send_flags=IBV_SEND_SIGNALED; - ib_wr->sq_wr_list[wr_index].wr_id =(uint64_t)nthread_counter_increment(&nnti_wrmap_counter); - ib_wr->sq_wr_list[wr_index].imm_data =hash6432shift((uint64_t)src_remote_addr->NNTI_remote_addr_t_u.ib.buf); - ib_wr->sq_wr_list[wr_index].next =NULL; // RAOLDFI ADDED - - log_debug(nnti_debug_level, "ib_wr->sq_wr_list[%d].sg_list[0].addr=0x%lX ; ib_wr->sq_wr_list[%d].sg_list[0].length=%u ; ib_wr->sq_wr_list[%d].sg_list[0].lkey=0x%X", - wr_index, ib_wr->sq_wr_list[wr_index].sg_list[0].addr, wr_index, ib_wr->sq_wr_list[wr_index].sg_list[0].length, wr_index, ib_wr->sq_wr_list[wr_index].sg_list[0].lkey); - log_debug(nnti_debug_level, "ib_wr->sq_wr_list[%d].wr.rdma._remote_addr=0x%lX ; ib_wr->sq_wr_list[%d].wr.rdma.rkey=0x%X", - wr_index, ib_wr->sq_wr_list[wr_index].wr.rdma.remote_addr, wr_index, ib_wr->sq_wr_list[wr_index].wr.rdma.rkey); - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->sq_wr_list[wr_index].wr_id, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->sq_wr_list[wr_index].wr_id) == wrmap.end()); - wrmap[ib_wr->sq_wr_list[wr_index].wr_id] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - if (src_params.segment_remaining < dst_params.segment_remaining) { - // the remaining source segment fits in the remaining destination segment with extra space - dst_params.segment_offset += src_params.segment_remaining; - dst_params.segment_remaining -= src_params.segment_remaining; - - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else if (src_params.segment_remaining == dst_params.segment_remaining) { - // the remaining source segment fits EXACTLY in the remaining destination segment - src_remote_addr++; - src_params.segment_index++; - src_params.segment_offset=0; - src_params.segment_remaining=src_remote_addr->NNTI_remote_addr_t_u.ib.size; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } else { - // the remaining source segment DOES NOT fit in the remaining destination segment - src_params.segment_offset += dst_params.segment_remaining; - src_params.segment_remaining -= dst_params.segment_remaining; - - dst_remote_addr++; - dst_params.segment_index++; - dst_params.segment_offset=0; - dst_params.segment_remaining=dst_remote_addr->NNTI_remote_addr_t_u.ib.size; - - } - - wr_index++; - } - } - - if (config.use_rdma_target_ack) { - ib_wr->ack.op =IB_OP_GET_TARGET; - ib_wr->ack.offset=src_offset; - ib_wr->ack.length=src_length; - - if (!config.use_wr_pool) { - register_ack(ib_wr); - } - ib_wr->ack_sge.addr =(uint64_t)ib_wr->ack_mr->addr; - ib_wr->ack_sge.length=ib_wr->ack_mr->length; - ib_wr->ack_sge.lkey =ib_wr->ack_mr->lkey; - - ib_wr->ack_sq_wr.sg_list=&ib_wr->ack_sge; - ib_wr->ack_sq_wr.num_sge=1; - - ib_wr->ack_sq_wr.wr.rdma.rkey =src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_key; - ib_wr->ack_sq_wr.wr.rdma.remote_addr=src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_buf; - - ib_wr->ack_sq_wr.opcode =IBV_WR_RDMA_WRITE_WITH_IMM; - ib_wr->ack_sq_wr.send_flags=IBV_SEND_SIGNALED|IBV_SEND_FENCE; - ib_wr->sq_wr.wr_id =(uint64_t)ib_wr->key; - ib_wr->ack_sq_wr.imm_data =hash6432shift((uint64_t)src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - ib_wr->ack_sq_wr.next =NULL; // RAOLDFI ADDED - } - - ib_wr->last_op=IB_OP_GET_INITIATOR; - ib_wr->length=src_length; - ib_wr->offset=dest_offset; - - log_debug(debug_level, "getting from (%s, qp=%p, qpn=%lu)", - src_buffer_hdl->buffer_owner.url, - ib_wr->qp, - ib_wr->qpn); - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - nthread_lock(&ib_mem_hdl->wr_queue_lock); - ib_mem_hdl->wr_queue.push_back(ib_wr); - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - wr->transport_id =dest_buffer_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)dest_buffer_hdl; - wr->ops =NNTI_BOP_LOCAL_WRITE; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)ib_wr; - - for (int i=0;isq_wr_count;i++) { - log_debug(nnti_debug_level, "posting ib_wr->sq_wr_list[%d]=%p (key=%lx)", i, ib_wr->sq_wr_list[i], ib_wr->sq_wr_list[i].wr_id); - trios_start_timer(call_time); - if (ibv_post_send_wrapper(ib_wr->qp, &ib_wr->sq_wr_list[i], &bad_wr)) { - log_error(nnti_debug_level, "failed to post send: %s", strerror(errno)); - rc=NNTI_EIO; - } - trios_stop_timer("NNTI_ib_get - ibv_post_send", call_time); - - if (config.use_rdma_target_ack) { - send_ack(ib_wr); - } - - // print_wr(ib_wr); - } - - if (ib_wr->sge_list != &ib_wr->sge) { - free(ib_wr->sge_list); - } - if (ib_wr->sq_wr_list != &ib_wr->sq_wr) { - free(ib_wr->sq_wr_list); - } - - log_debug(nnti_debug_level, "exit"); - - trios_stop_timer("NNTI_ib_get - total", total_time); - - return(rc); -} - - -/** - * @brief Transfer data to a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to put. - * \param[in] src_length The number of bytes to put. - * \param[in] dest_buffer_list A list of buffers to put the data into. - * \param[in] dest_count The number of destination buffers. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_ib_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *ib_wr) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -/** - * @brief Transfer data from a peer. - * - * \param[in] src_buffer_list A list of buffers containing the data to get. - * \param[in] src_length The number of bytes to get. - * \param[in] src_count The number of source buffers. - * \param[in] dest_buffer_hdl A buffer to get the data into. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_ib_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *ib_wr) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_ib_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_ib_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value) -{ - nthread_lock(&transport_global_data.atomics_lock); - *value = transport_global_data.atomics[local_atomic]; - nthread_unlock(&transport_global_data.atomics_lock); - - return NNTI_OK; -} - - -NNTI_result_t NNTI_ib_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - trios_declare_timer(call_time); - - struct ibv_send_wr *bad_wr; - - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - - log_level debug_level=nnti_debug_level; - - if (config.use_wr_pool) { - ib_wr=wr_pool_sendrecv_pop(); - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - log_debug(nnti_debug_level, "allocated ib_wr (wr=%p ; ib_wr=%p)", wr, ib_wr); - nthread_lock_init(&ib_wr->lock); - } - assert(ib_wr); - - ib_wr->conn = get_conn_peer(peer_hdl); - assert(ib_wr->conn); - - ib_wr->nnti_wr = wr; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - ib_wr->qp =ib_wr->conn->data_qp.qp; - ib_wr->qpn =(uint64_t)ib_wr->conn->data_qp.qpn; - ib_wr->peer_qpn =(uint64_t)ib_wr->conn->data_qp.peer_qpn; - - ib_wr->last_op=IB_OP_FETCH_ADD; - - ib_wr->sq_wr_list =&ib_wr->sq_wr; - ib_wr->sq_wr_count=1; - - ib_wr->sq_wr.wr.atomic.rkey =ib_wr->conn->atomics_rkey; - ib_wr->sq_wr.wr.atomic.remote_addr=ib_wr->conn->atomics_addr+(target_atomic*sizeof(int64_t)); - ib_wr->sq_wr.wr.atomic.compare_add=operand; - - ib_wr->sq_wr.opcode =IBV_WR_ATOMIC_FETCH_AND_ADD; - ib_wr->sq_wr.send_flags=IBV_SEND_SIGNALED; - - ib_wr->sge_count=1; - ib_wr->sge_list=&ib_wr->sge; - ib_wr->sge_list[0].addr =(uint64_t)&transport_global_data.atomics[result_atomic]; - ib_wr->sge_list[0].length=sizeof(int64_t); - ib_wr->sge_list[0].lkey =transport_global_data.atomics_mr->lkey; - - ib_wr->sq_wr.wr_id = (uint64_t)ib_wr->key; - ib_wr->sq_wr.next = NULL; - ib_wr->sq_wr.sg_list = ib_wr->sge_list; - ib_wr->sq_wr.num_sge = ib_wr->sge_count; - - - log_debug(nnti_debug_level, "sending to (%s, qp=%p, qpn=%d, sge.addr=%p, sge.length=%llu, sq_wr.ib_wr.rdma.rkey=0x%x, sq_wr.ib_wr.rdma.remote_addr=%p)", - peer_hdl->url, - ib_wr->qp, - ib_wr->qpn, - (void *) ib_wr->sge.addr, - (uint64_t)ib_wr->sge.length, - ib_wr->sq_wr.wr.atomic.rkey, - (void *) ib_wr->sq_wr.wr.atomic.remote_addr); - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - wr->transport_id =trans_hdl->id; - wr->reg_buf =(NNTI_buffer_t*)NULL; - wr->ops =NNTI_BOP_ATOMICS; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)ib_wr; - - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - trios_start_timer(call_time); - if (ibv_post_send_wrapper(ib_wr->qp, &ib_wr->sq_wr, &bad_wr)) { - log_error(nnti_debug_level, "failed to post send: %s", strerror(errno)); - rc=NNTI_EIO; - } - trios_stop_timer("NNTI_ib_send - ibv_post_send", call_time); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -NNTI_result_t NNTI_ib_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - - trios_declare_timer(call_time); - - struct ibv_send_wr *bad_wr; - - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - - log_level debug_level=nnti_debug_level; - - if (config.use_wr_pool) { - ib_wr=wr_pool_sendrecv_pop(); - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - log_debug(nnti_debug_level, "allocated ib_wr (wr=%p ; ib_wr=%p)", wr, ib_wr); - nthread_lock_init(&ib_wr->lock); - } - assert(ib_wr); - - ib_wr->conn = get_conn_peer(peer_hdl); - assert(ib_wr->conn); - - ib_wr->nnti_wr = wr; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->state=NNTI_IB_WR_STATE_STARTED; - - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - ib_wr->qp =ib_wr->conn->data_qp.qp; - ib_wr->qpn =(uint64_t)ib_wr->conn->data_qp.qpn; - ib_wr->peer_qpn =(uint64_t)ib_wr->conn->data_qp.peer_qpn; - - ib_wr->last_op=IB_OP_COMPARE_SWAP; - - ib_wr->sq_wr_list =&ib_wr->sq_wr; - ib_wr->sq_wr_count=1; - - ib_wr->sq_wr.wr.atomic.rkey =ib_wr->conn->atomics_rkey; - ib_wr->sq_wr.wr.atomic.remote_addr=ib_wr->conn->atomics_addr+(target_atomic*sizeof(int64_t)); - ib_wr->sq_wr.wr.atomic.compare_add=compare_operand; - ib_wr->sq_wr.wr.atomic.swap =swap_operand; - - ib_wr->sq_wr.opcode =IBV_WR_ATOMIC_CMP_AND_SWP; - ib_wr->sq_wr.send_flags=IBV_SEND_SIGNALED; - - ib_wr->sge_count=1; - ib_wr->sge_list=&ib_wr->sge; - ib_wr->sge_list[0].addr =(uint64_t)&transport_global_data.atomics[result_atomic]; - ib_wr->sge_list[0].length=sizeof(int64_t); - ib_wr->sge_list[0].lkey =transport_global_data.atomics_mr->lkey; - - ib_wr->sq_wr.wr_id = (uint64_t)ib_wr->key; - ib_wr->sq_wr.next = NULL; - ib_wr->sq_wr.sg_list = ib_wr->sge_list; - ib_wr->sq_wr.num_sge = ib_wr->sge_count; - - - log_debug(nnti_debug_level, "sending to (%s, qp=%p, qpn=%d, sge.addr=%p, sge.length=%llu, sq_wr.ib_wr.rdma.rkey=0x%x, sq_wr.ib_wr.rdma.remote_addr=%p)", - peer_hdl->url, - ib_wr->qp, - ib_wr->qpn, - (void *) ib_wr->sge.addr, - (uint64_t)ib_wr->sge.length, - ib_wr->sq_wr.wr.atomic.rkey, - (void *) ib_wr->sq_wr.wr.atomic.remote_addr); - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - wr->transport_id =trans_hdl->id; - wr->reg_buf =(NNTI_buffer_t*)NULL; - wr->ops =NNTI_BOP_ATOMICS; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)ib_wr; - - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - trios_start_timer(call_time); - if (ibv_post_send_wrapper(ib_wr->qp, &ib_wr->sq_wr, &bad_wr)) { - log_error(nnti_debug_level, "failed to post send: %s", strerror(errno)); - rc=NNTI_EIO; - } - trios_stop_timer("NNTI_ib_send - ibv_post_send", call_time); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - - -/** - * @brief Create a receive work request that can be used to wait for buffer - * operations to complete. - * - */ -NNTI_result_t NNTI_ib_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr) -{ - wr_queue_iter_t iter; - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p ; wr=%p)", reg_buf, wr); - - ib_mem_hdl=IB_MEM_HDL(reg_buf); - assert(ib_mem_hdl); - - nthread_lock(&ib_mem_hdl->wr_queue_lock); - - for ( iter=ib_mem_hdl->wr_queue.begin() ; iter != ib_mem_hdl->wr_queue.end() ; ++iter ) { - if ((*iter)->nnti_wr == NULL) { - ib_wr=*iter; - break; - } - } - assert(ib_wr); - - wr->transport_private=(uint64_t)ib_wr; - ib_wr->nnti_wr=wr; - - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - wr->transport_id =reg_buf->transport_id; - wr->reg_buf =reg_buf; - wr->ops =reg_buf->ops; - wr->result =NNTI_OK; - - log_debug(nnti_debug_level, "assigned ib_work_request(%p) to NNTI_work_request(%p)", ib_wr, wr); - - log_debug(nnti_debug_level, "exit (reg_buf=%p ; wr=%p)", reg_buf, wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from a previous receive - * and prepares it for reuse. - * - */ -NNTI_result_t NNTI_ib_clear_work_request ( - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - wr->result =NNTI_OK; - wr->transport_private=NULL; - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from reg_buf. - * - */ -NNTI_result_t NNTI_ib_destroy_work_request ( - NNTI_work_request_t *wr) -{ - ib_memory_handle *ib_mem_hdl; - ib_work_request *ib_wr; - - log_debug(nnti_debug_level, "enter (wr=%p ; ib_wr=%p)", wr, ib_wr); - - ib_mem_hdl=IB_MEM_HDL(wr->reg_buf); - assert(ib_mem_hdl); - ib_wr=IB_WORK_REQUEST(wr); - - if (!ib_wr) { - return(NNTI_OK); - } - - if ((ib_wr->last_op == IB_OP_SEND_REQUEST) || - (ib_wr->last_op == IB_OP_SEND_BUFFER) || - (ib_wr->last_op == IB_OP_GET_INITIATOR) || - (ib_wr->last_op == IB_OP_PUT_INITIATOR)) { - // work requrests from initiator buffers are cleaned up in NNTI_ib_wait*(). nothing to do here. - return(NNTI_OK); - } - - if (ib_wr->last_op == IB_OP_NEW_REQUEST) { - if (ib_wr->state == NNTI_IB_WR_STATE_WAIT_COMPLETE) { - repost_recv_work_request(wr, ib_wr); - - nthread_lock(&ib_mem_hdl->wr_queue_lock); - wr_queue_iter_t q_victim=find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), ib_wr); - if (q_victim != ib_mem_hdl->wr_queue.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p from the wr_queue", ib_wr); - ib_mem_hdl->wr_queue.erase(q_victim); - } - assert(find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), ib_wr) == ib_mem_hdl->wr_queue.end()); - - ib_mem_hdl->wr_queue.push_back(ib_wr); - - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - } - ib_wr->nnti_wr=NULL; - - } else if (ib_wr->last_op == IB_OP_RECEIVE) { - if (ib_wr->state == NNTI_IB_WR_STATE_WAIT_COMPLETE) { - repost_recv_work_request(wr, ib_wr); - - nthread_lock(&ib_mem_hdl->wr_queue_lock); - wr_queue_iter_t q_victim=find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), ib_wr); - if (q_victim != ib_mem_hdl->wr_queue.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p from the wr_queue", ib_wr); - ib_mem_hdl->wr_queue.erase(q_victim); - } - assert(find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), ib_wr) == ib_mem_hdl->wr_queue.end()); - - ib_mem_hdl->wr_queue.push_back(ib_wr); - - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - } - ib_wr->nnti_wr=NULL; - - } - - wr->transport_id =NNTI_TRANSPORT_NULL; - wr->reg_buf =NULL; - wr->ops =(NNTI_buf_ops_t)0; - wr->result =NNTI_OK; - wr->transport_private=NULL; - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Attempts to cancel an NNTI opertion. - * - */ -NNTI_result_t NNTI_ib_cancel ( - NNTI_work_request_t *wr) -{ - NNTI_result_t rc=NNTI_OK; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - ib_wr=IB_WORK_REQUEST(wr); - assert(ib_wr); - - nthread_lock(&ib_wr->lock); - if ((ib_wr->state == NNTI_IB_WR_STATE_STARTED) || (ib_wr->state == NNTI_IB_WR_STATE_POSTED)) { - ib_wr->state = NNTI_IB_WR_STATE_CANCELING; - } else { - log_warn(nnti_debug_level, "wr=%p is not in progress (current state is %d). Cannot cancel.", wr, ib_wr->state); - rc=NNTI_EINVAL; - } - nthread_unlock(&ib_wr->lock); - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return rc; -} - - -/** - * @brief Attempts to cancel a list of NNTI opertions. - * - */ -NNTI_result_t NNTI_ib_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - NNTI_result_t rc=NNTI_OK; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - for (int i=0;ilock); - if ((ib_wr->state == NNTI_IB_WR_STATE_STARTED) || (ib_wr->state == NNTI_IB_WR_STATE_POSTED)) { - ib_wr->state = NNTI_IB_WR_STATE_CANCELING; - } else { - log_warn(nnti_debug_level, "wr_list[%d]=%p is not in progress (current state is %d). Cannot cancel.", i, wr_list[i], ib_wr->state); - rc=NNTI_EINVAL; - } - nthread_unlock(&ib_wr->lock); - } - - log_debug(nnti_debug_level, "exit"); - - return rc; -} - - -/** - * @brief Interrupts NNTI_wait*() - * - */ -NNTI_result_t NNTI_ib_interrupt ( - const NNTI_transport_t *trans_hdl) -{ - uint32_t dummy=0xAAAAAAAA; - - log_debug(nnti_debug_level, "enter"); - - write(transport_global_data.interrupt_pipe[1], &dummy, 4); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_OK; -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t NNTI_ib_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status) -{ - NNTI_result_t nnti_rc=NNTI_OK; - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - const NNTI_buffer_t *wait_buf=NULL; - - int ibv_rc=0; - NNTI_result_t rc; - long elapsed_time = 0; - long timeout_per_call; - - log_level debug_level=nnti_debug_level; - - log_level old_log_level=logger_get_default_level(); - - long entry_time=trios_get_time_ms(); - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - struct ibv_wc wc; - - log_debug(debug_level, "enter (wr=%p ; ib_wr=%p ; timeout=%d)", wr, IB_WORK_REQUEST(wr), timeout); - - trios_start_timer(total_time); - - assert(wr); - assert(status); - - if (logging_debug(debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "wr->reg_buf", - "start of NNTI_ib_wait", wr->reg_buf); - } - - ib_wr=IB_WORK_REQUEST(wr); - assert(ib_wr); - - if (is_wr_canceling(ib_wr)) { - cancel_wr(ib_wr); - } else if (is_wr_complete(ib_wr) == TRUE) { - log_debug(debug_level, "wr already complete (wr=%p ; ib_wr=%p)", wr, IB_WORK_REQUEST(wr)); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "wr NOT complete (wr=%p ; ib_wr=%p)", wr, IB_WORK_REQUEST(wr)); - - trios_start_timer(call_time); - - while (1) { - rc=progress(timeout-elapsed_time, &wr, 1); - - elapsed_time = (trios_get_time_ms() - entry_time); - - /* case 1: success */ - if (rc == NNTI_OK) { -// logger_set_default_level(old_log_level); - log_debug(debug_level, "progress() successful..."); - nnti_rc = rc; - } - /* case 2: timed out */ - else if (rc==NNTI_ETIMEDOUT) { - log_debug(debug_level, "progress() timed out..."); -// logger_set_default_level(LOG_OFF); - } - /* case 3: interrupted */ - else if (rc==NNTI_EINTR) { - log_debug(debug_level, "progress() interrupted..."); -// logger_set_default_level(LOG_OFF); - nnti_rc = rc; - break; - } - /* case 4: failure */ - else { -// logger_set_default_level(old_log_level); - log_debug(debug_level, "progress() failed: %s", strerror(errno)); - nnti_rc = rc; - break; - } - - if (is_wr_complete(ib_wr) == TRUE) { - log_debug(debug_level, "wr completed (wr=%p ; ib_wr=%p)", wr, IB_WORK_REQUEST(wr)); - nnti_rc = NNTI_OK; - break; - } - - /* if the caller asked for a legitimate timeout, we need to exit */ - if ((timeout >= 0) && (elapsed_time >= timeout)) { - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - nnti_rc=NNTI_ECANCELED; - break; - } - } - - trios_stop_timer("NNTI_ib_wait - wr complete", call_time); - } - - trios_start_timer(call_time); - create_status(wr, ib_wr, nnti_rc, status); - trios_stop_timer("NNTI_ib_wait - create_status", call_time); - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_ib_wait", status); - } - - if (is_wr_complete(ib_wr)) { - - trios_start_timer(call_time); - - ib_wr->state=NNTI_IB_WR_STATE_WAIT_COMPLETE; - - if (ib_wr->nnti_wr->ops == NNTI_BOP_ATOMICS) { - nthread_lock(&nnti_wrmap_lock); - wrmap_iter_t m_victim=wrmap.find(ib_wr->key); - if (m_victim != wrmap.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p (key=%lx) from the wrmap", ib_wr, ib_wr->key); - wrmap.erase(m_victim); - } - nthread_unlock(&nnti_wrmap_lock); - - if (config.use_wr_pool) { - wr_pool_sendrecv_push(ib_wr); - } else { - if (config.use_rdma_target_ack) { - unregister_ack(ib_wr); - } - log_debug(nnti_debug_level, "freeing ib_wr=%p", ib_wr); - free(ib_wr); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr->transport_private=(uint64_t)NULL; - - } else { - ib_mem_hdl=IB_MEM_HDL(wr->reg_buf); - assert(ib_mem_hdl); - - if ((ib_wr->last_op == IB_OP_NEW_REQUEST) || (ib_wr->last_op == IB_OP_RECEIVE)) { - // defer cleanup to NNTI_ib_destroy_work_request() - } - else if ((ib_wr->last_op == IB_OP_GET_TARGET) || - (ib_wr->last_op == IB_OP_PUT_TARGET)) { - if (config.use_rdma_target_ack) { - repost_ack_recv_work_request(wr, ib_wr); - } - } - else { - nthread_lock(&ib_mem_hdl->wr_queue_lock); - wr_queue_iter_t q_victim=find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), ib_wr); - if (q_victim != ib_mem_hdl->wr_queue.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p from the wr_queue", ib_wr); - ib_mem_hdl->wr_queue.erase(q_victim); - } - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - nthread_lock(&nnti_wrmap_lock); - wrmap_iter_t m_victim=wrmap.find(ib_wr->key); - if (m_victim != wrmap.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p (key=%lx) from the wrmap", ib_wr, ib_wr->key); - wrmap.erase(m_victim); - } - nthread_unlock(&nnti_wrmap_lock); - - if (config.use_wr_pool) { - wr_pool_sendrecv_push(ib_wr); - } else { - if (config.use_rdma_target_ack) { - unregister_ack(ib_wr); - } - log_debug(nnti_debug_level, "freeing ib_wr=%p", ib_wr); - free(ib_wr); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr->transport_private=(uint64_t)NULL; - } - } - - trios_stop_timer("NNTI_ib_wait - wr cleanup", call_time); - } - - trios_stop_timer("NNTI_ib_wait", total_time); - - log_debug(debug_level, "exit (wr=%p ; ib_wr=%p)", wr, IB_WORK_REQUEST(wr)); - - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on any buffer in buf_list to complete. - * - * Wait for remote_op on any buffer in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the request queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_ib_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status) -{ - NNTI_result_t nnti_rc=NNTI_OK; - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - const NNTI_buffer_t *wait_buf=NULL; - - int ibv_rc=0; - NNTI_result_t rc; - long elapsed_time = 0; - long timeout_per_call; - - log_level debug_level=nnti_debug_level; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - struct ibv_wc wc; - - long entry_time=trios_get_time_ms(); - - log_debug(debug_level, "enter"); - - trios_start_timer(total_time); - - assert(wr_list); - assert(wr_count > 0); - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_ib_wait(wr_list[0], timeout, status); - *which=0; - goto cleanup; - } - - if (wr_count > 1) { - for (uint32_t i=0;istate=NNTI_IB_WR_STATE_WAIT_COMPLETE; - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "wr NOT complete (wr_list=%p)", wr_list); - - while (1) { - rc=progress(timeout-elapsed_time, wr_list, wr_count); - - elapsed_time = (trios_get_time_ms() - entry_time); - - /* case 1: success */ - if (rc == NNTI_OK) { -// logger_set_default_level(old_log_level); - log_debug(debug_level, "progress() successful..."); - nnti_rc = NNTI_OK; - } - /* case 2: timed out */ - else if (rc==NNTI_ETIMEDOUT) { - log_debug(debug_level, "progress() timed out..."); -// logger_set_default_level(LOG_OFF); - } - /* case 3: interrupted */ - else if (rc==NNTI_EINTR) { - log_debug(debug_level, "progress() interrupted..."); -// logger_set_default_level(LOG_OFF); - nnti_rc=NNTI_EINTR; - break; - } - /* case 4: failure */ - else { -// logger_set_default_level(old_log_level); - log_error(debug_level, "progress() failed: %s", strerror(errno)); - nnti_rc = NNTI_EIO; - break; - } - - if (is_any_wr_complete(wr_list, wr_count, which) == TRUE) { - log_debug(debug_level, "wr completed (which=%u, wr_list[%d]=%p, ib_wr=%p)", *which, *which, wr_list[*which], IB_WORK_REQUEST(wr_list[*which])); - IB_WORK_REQUEST(wr_list[*which])->state=NNTI_IB_WR_STATE_WAIT_COMPLETE; - nnti_rc = NNTI_OK; - break; - } - - /* if the caller asked for a legitimate timeout, we need to exit */ - if ((timeout >= 0) && (elapsed_time >= timeout)) { - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - nnti_rc=NNTI_ECANCELED; - break; - } - } - } - - create_status(wr_list[*which], IB_WORK_REQUEST(wr_list[*which]), nnti_rc, status); - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_ib_waitany", status); - } - - if (is_wr_complete(IB_WORK_REQUEST(wr_list[*which]))) { - - IB_WORK_REQUEST(wr_list[*which])->state=NNTI_IB_WR_STATE_WAIT_COMPLETE; - - ib_mem_hdl=IB_MEM_HDL(wr_list[*which]->reg_buf); - assert(ib_mem_hdl); - - if ((ib_wr->last_op == IB_OP_NEW_REQUEST) || (ib_wr->last_op == IB_OP_RECEIVE)) { - // defer cleanup to NNTI_ib_destroy_work_request() - } - else if ((ib_wr->last_op == IB_OP_GET_TARGET) || - (ib_wr->last_op == IB_OP_PUT_TARGET)) { - if (config.use_rdma_target_ack) { - repost_ack_recv_work_request(wr_list[*which], IB_WORK_REQUEST(wr_list[*which])); - } - } - else { - nthread_lock(&ib_mem_hdl->wr_queue_lock); - wr_queue_iter_t q_victim=find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), IB_WORK_REQUEST(wr_list[*which])); - if (q_victim != ib_mem_hdl->wr_queue.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p from the wr_queue", IB_WORK_REQUEST(wr_list[*which])); - ib_mem_hdl->wr_queue.erase(q_victim); - } - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - nthread_lock(&nnti_wrmap_lock); - wrmap_iter_t m_victim=wrmap.find(IB_WORK_REQUEST(wr_list[*which])->key); - if (m_victim != wrmap.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p (key=%lx) from the wrmap", IB_WORK_REQUEST(wr_list[*which]), IB_WORK_REQUEST(wr_list[*which])->key); - wrmap.erase(m_victim); - } - nthread_unlock(&nnti_wrmap_lock); - - if (config.use_wr_pool) { - wr_pool_sendrecv_push(IB_WORK_REQUEST(wr_list[*which])); - } else { - if (config.use_rdma_target_ack) { - unregister_ack(IB_WORK_REQUEST(wr_list[*which])); - } - log_debug(nnti_debug_level, "freeing ib_wr=%p", IB_WORK_REQUEST(wr_list[*which])); - free(IB_WORK_REQUEST(wr_list[*which])); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr_list[*which]->transport_private=(uint64_t)NULL; - } - } - -cleanup: - trios_stop_timer("NNTI_ib_waitany", total_time); - - log_debug(debug_level, "exit"); - - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on all buffers in buf_list to complete. - * - * Wait for remote_op on all buffers in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the receive queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_ib_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status) -{ - NNTI_result_t nnti_rc=NNTI_OK; - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - const NNTI_buffer_t *wait_buf=NULL; - - int ibv_rc=0; - NNTI_result_t rc; - long elapsed_time = 0; - long timeout_per_call; - - log_level debug_level=nnti_debug_level; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - struct ibv_wc wc; - - long entry_time=trios_get_time_ms(); - - log_debug(debug_level, "enter"); - - trios_start_timer(total_time); - - assert(wr_list); - assert(wr_count > 0); - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_ib_wait(wr_list[0], timeout, status[0]); - goto cleanup; - } - - if (wr_count > 1) { - for (uint32_t i=0;i= 0) && (elapsed_time >= timeout)) { - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - nnti_rc=NNTI_ECANCELED; - break; - } - } - } - - for (uint32_t i=0;istate=NNTI_IB_WR_STATE_WAIT_COMPLETE; - - ib_mem_hdl=IB_MEM_HDL(wr_list[i]->reg_buf); - assert(ib_mem_hdl); - - if ((ib_wr->last_op == IB_OP_NEW_REQUEST) || (ib_wr->last_op == IB_OP_RECEIVE)) { - // defer cleanup to NNTI_ib_destroy_work_request() - } - else if ((ib_wr->last_op == IB_OP_GET_TARGET) || - (ib_wr->last_op == IB_OP_PUT_TARGET)) { - if (config.use_rdma_target_ack) { - repost_ack_recv_work_request(wr_list[i], IB_WORK_REQUEST(wr_list[i])); - } - } - else { - nthread_lock(&ib_mem_hdl->wr_queue_lock); - wr_queue_iter_t q_victim=find(ib_mem_hdl->wr_queue.begin(), ib_mem_hdl->wr_queue.end(), IB_WORK_REQUEST(wr_list[i])); - if (q_victim != ib_mem_hdl->wr_queue.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p from the wr_queue", IB_WORK_REQUEST(wr_list[i])); - ib_mem_hdl->wr_queue.erase(q_victim); - } - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - nthread_lock(&nnti_wrmap_lock); - wrmap_iter_t m_victim=wrmap.find(IB_WORK_REQUEST(wr_list[i])->key); - if (m_victim != wrmap.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p (key=%lx) from the wrmap", IB_WORK_REQUEST(wr_list[i]), IB_WORK_REQUEST(wr_list[i])->key); - wrmap.erase(m_victim); - } - nthread_unlock(&nnti_wrmap_lock); - - if (config.use_wr_pool) { - wr_pool_sendrecv_push(IB_WORK_REQUEST(wr_list[i])); - } else { - if (config.use_rdma_target_ack) { - unregister_ack(IB_WORK_REQUEST(wr_list[i])); - } - log_debug(nnti_debug_level, "freeing ib_wr=%p", IB_WORK_REQUEST(wr_list[i])); - free(IB_WORK_REQUEST(wr_list[i])); - } - /* - * This work request (wr) has reached a completed (final) state. wr is reset here. - */ - wr_list[i]->transport_private=(uint64_t)NULL; - } - } - } - -cleanup: - trios_stop_timer("NNTI_ib_waitall", total_time); - - log_debug(debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t NNTI_ib_fini ( - const NNTI_transport_t *trans_hdl) -{ - NNTI_result_t rc=NNTI_OK;; - - log_debug(nnti_debug_level, "enter"); - - close_all_conn(); - - if (config.use_wr_pool) { - rc=wr_pool_fini(); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "wr_pool_fini() failed: %d", rc); - rc=NNTI_EINVAL; - } - } - - close(transport_global_data.listen_sock); - transport_global_data.listen_name[0]='\0'; - transport_global_data.listen_addr=0; - transport_global_data.listen_port=0; - - ibv_destroy_comp_channel_wrapper(transport_global_data.data_comp_channel); - ibv_destroy_cq_wrapper(transport_global_data.data_cq); - ibv_destroy_srq_wrapper(transport_global_data.data_srq); - - ibv_destroy_comp_channel_wrapper(transport_global_data.req_comp_channel); - ibv_destroy_cq_wrapper(transport_global_data.req_cq); - ibv_destroy_srq_wrapper(transport_global_data.req_srq); - - ibv_dealloc_pd_wrapper(transport_global_data.pd); - - ibv_close_device_wrapper(transport_global_data.ctx); - - nthread_lock_fini(&nnti_ib_lock); - - nthread_lock_fini(&nnti_progress_lock); - nthread_cond_fini(&nnti_progress_cond); - - nthread_lock_fini(&nnti_conn_peer_lock); - nthread_lock_fini(&nnti_conn_qpn_lock); - nthread_lock_fini(&nnti_buf_bufhash_lock); - nthread_lock_fini(&transport_global_data.atomics_lock); - nthread_lock_fini(&nnti_wr_pool_lock); - - ib_initialized=false; - - log_debug(nnti_debug_level, "exit"); - - return(NNTI_OK); -} - - - - - -static NNTI_result_t setup_request_channel(void) -{ - int flags; - - transport_global_data.req_comp_channel = ibv_create_comp_channel_wrapper(transport_global_data.ctx); - if (!transport_global_data.req_comp_channel) { - log_error(nnti_debug_level, "ibv_create_comp_channel failed"); - return NNTI_EIO; - } - transport_global_data.req_cq = ibv_create_cq_wrapper( - transport_global_data.ctx, - transport_global_data.cqe_count, - NULL, - transport_global_data.req_comp_channel, - 0); - if (!transport_global_data.req_cq) { - log_error(nnti_debug_level, "ibv_create_cq failed"); - return NNTI_EIO; - } - - transport_global_data.req_srq_count=0; - - struct ibv_srq_init_attr srq_attr; - memset(&srq_attr, 0, sizeof(srq_attr)); // initialize to avoid valgrind uninitialized warning - srq_attr.attr.max_wr = transport_global_data.srq_count; - srq_attr.attr.max_sge = transport_global_data.sge_count; - - transport_global_data.req_srq = ibv_create_srq_wrapper(transport_global_data.pd, &srq_attr); - if (!transport_global_data.req_srq) { - log_error(nnti_debug_level, "ibv_create_srq failed"); - return NNTI_EIO; - } - - if (ibv_req_notify_cq_wrapper(transport_global_data.req_cq, 0)) { - log_error(nnti_debug_level, "ibv_req_notify_cq failed"); - return NNTI_EIO; - } - - /* use non-blocking IO on the async fd and completion fd */ - flags = fcntl(transport_global_data.ctx->async_fd, F_GETFL); - if (flags < 0) { - log_error(nnti_debug_level, "failed to get async_fd flags"); - return NNTI_EIO; - } - if (fcntl(transport_global_data.ctx->async_fd, F_SETFL, flags | O_NONBLOCK) < 0) { - log_error(nnti_debug_level, "failed to set async_fd to nonblocking"); - return NNTI_EIO; - } - - flags = fcntl(transport_global_data.req_comp_channel->fd, F_GETFL); - if (flags < 0) { - log_error(nnti_debug_level, "failed to get completion fd flags"); - return NNTI_EIO; - } - if (fcntl(transport_global_data.req_comp_channel->fd, F_SETFL, flags | O_NONBLOCK) < 0) { - log_error(nnti_debug_level, "failed to set completion fd to nonblocking"); - return NNTI_EIO; - } - - - return(NNTI_OK); -} - -static NNTI_result_t setup_data_channel(void) -{ - int flags; - - transport_global_data.data_comp_channel = ibv_create_comp_channel_wrapper(transport_global_data.ctx); - if (!transport_global_data.data_comp_channel) { - log_error(nnti_debug_level, "ibv_create_comp_channel failed"); - return NNTI_EIO; - } - transport_global_data.data_cq = ibv_create_cq_wrapper( - transport_global_data.ctx, - transport_global_data.cqe_count, - NULL, - transport_global_data.data_comp_channel, - 0); - if (!transport_global_data.data_cq) { - log_error(nnti_debug_level, "ibv_create_cq failed"); - return NNTI_EIO; - } - - transport_global_data.data_srq_count=0; - - struct ibv_srq_init_attr srq_attr; - memset(&srq_attr, 0, sizeof(srq_attr)); // initialize to avoid valgrind warning - srq_attr.attr.max_wr = transport_global_data.srq_count; - srq_attr.attr.max_sge = transport_global_data.sge_count; - - transport_global_data.data_srq = ibv_create_srq_wrapper(transport_global_data.pd, &srq_attr); - if (!transport_global_data.data_srq) { - log_error(nnti_debug_level, "ibv_create_srq failed"); - return NNTI_EIO; - } - - if (ibv_req_notify_cq_wrapper(transport_global_data.data_cq, 0)) { - log_error(nnti_debug_level, "ibv_req_notify_cq failed"); - return NNTI_EIO; - } - - /* use non-blocking IO on the async fd and completion fd */ - flags = fcntl(transport_global_data.ctx->async_fd, F_GETFL); - if (flags < 0) { - log_error(nnti_debug_level, "failed to get async_fd flags"); - return NNTI_EIO; - } - if (fcntl(transport_global_data.ctx->async_fd, F_SETFL, flags | O_NONBLOCK) < 0) { - log_error(nnti_debug_level, "failed to set async_fd to nonblocking"); - return NNTI_EIO; - } - - flags = fcntl(transport_global_data.data_comp_channel->fd, F_GETFL); - if (flags < 0) { - log_error(nnti_debug_level, "failed to get completion fd flags"); - return NNTI_EIO; - } - if (fcntl(transport_global_data.data_comp_channel->fd, F_SETFL, flags | O_NONBLOCK) < 0) { - log_error(nnti_debug_level, "failed to set completion fd to nonblocking"); - return NNTI_EIO; - } - - - return(NNTI_OK); -} - -static NNTI_result_t setup_interrupt_pipe(void) -{ - int rc=0; - int flags; - - rc=pipe(transport_global_data.interrupt_pipe); - if (rc < 0) { - log_error(nnti_debug_level, "pipe() failed: %s", strerror(errno)); - return NNTI_EIO; - } - - /* use non-blocking IO on the read side of the interrupt pipe */ - flags = fcntl(transport_global_data.interrupt_pipe[0], F_GETFL); - if (flags < 0) { - log_error(nnti_debug_level, "failed to get interrupt_pipe flags: %s", strerror(errno)); - return NNTI_EIO; - } - if (fcntl(transport_global_data.interrupt_pipe[0], F_SETFL, flags | O_NONBLOCK) < 0) { - log_error(nnti_debug_level, "failed to set interrupt_pipe to nonblocking: %s", strerror(errno)); - return NNTI_EIO; - } - - /* use non-blocking IO on the write side of the interrupt pipe */ - flags = fcntl(transport_global_data.interrupt_pipe[1], F_GETFL); - if (flags < 0) { - log_error(nnti_debug_level, "failed to get interrupt_pipe flags: %s", strerror(errno)); - return NNTI_EIO; - } - if (fcntl(transport_global_data.interrupt_pipe[1], F_SETFL, flags | O_NONBLOCK) < 0) { - log_error(nnti_debug_level, "failed to set interrupt_pipe to nonblocking: %s", strerror(errno)); - return NNTI_EIO; - } - - return(NNTI_OK); -} - -static NNTI_result_t setup_atomics(void) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - - trios_declare_timer(callTime); - - struct ibv_mr *mr=NULL; - - uint32_t atomics_bytes; - - int flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_ATOMIC; - - log_debug(nnti_debug_level, "enter"); - - atomics_bytes=config.min_atomics_vars * sizeof(int64_t); - trios_start_timer(callTime); - transport_global_data.atomics=(int64_t*)aligned_malloc(atomics_bytes); - if (transport_global_data.atomics == NULL) { - return(NNTI_ENOMEM); - } - memset(transport_global_data.atomics, 0, atomics_bytes); - trios_stop_timer("malloc and memset", callTime); - - trios_start_timer(callTime); - mr = ibv_reg_mr_wrapper(transport_global_data.pd, transport_global_data.atomics, atomics_bytes, flags); - if (!mr) { - log_error( - nnti_debug_level, - "failed to register memory region (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ " - "| IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_ATOMIC): %s", - strerror(errno)); - return(NNTI_ENOMEM); - } - trios_stop_timer("register", callTime); - - transport_global_data.atomics_mr=mr; - - log_debug(nnti_debug_level, "exit (buf==%p, mr==%p, lkey %x, rkey %x)...", transport_global_data.atomics, mr, mr->lkey, mr->rkey); - - return(NNTI_OK); -} - -static void *aligned_malloc( - size_t size) -{ - int rc = 0; - void * ptr = NULL; - -#ifdef NO_MEMALIGN - ptr = malloc( size ); -#else - rc = posix_memalign( &ptr , 128 , size ); -#endif - - if (rc!=0) { - log_error(nnti_debug_level, "posix_memalign() failed: rc=%d (%s)", rc, strerror(errno)); - ptr=NULL; - } - - return ptr; -} - -static struct ibv_mr *register_memory_segment( - void *buf, - uint64_t len, - enum ibv_access_flags access) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - - trios_declare_timer(callTime); - - struct ibv_mr *mr=NULL; - - log_debug(nnti_debug_level, "enter buffer(%p) len(%d)", buf, len); - - if (config.use_mlock) { - trios_start_timer(callTime); - mlock(buf, len); - trios_stop_timer("mlock", callTime); - } - - trios_start_timer(callTime); - mr = ibv_reg_mr_wrapper(transport_global_data.pd, buf, len, access); - if (!mr) { - if (errno == EFAULT) { - log_debug(nnti_debug_level, "ibv_reg_mr failed with EFAULT. trying to register with IBV_ACCESS_REMOTE_READ."); - mr = ibv_reg_mr_wrapper(transport_global_data.pd, buf, len, IBV_ACCESS_REMOTE_READ); - if (!mr) { - log_error(nnti_debug_level, "failed to register memory region with IBV_ACCESS_REMOTE_READ: %s", strerror(errno)); - return(NULL); - } - } else { - log_error(nnti_debug_level, "failed to register memory region (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE): %s", strerror(errno)); - return(NULL); - } - } - trios_stop_timer("register", callTime); - - log_debug(nnti_debug_level, "exit (buf==%p, mr==%p, lkey %x, rkey %x)...", buf, mr, mr->lkey, mr->rkey); - - return(mr); -} - -static int register_ack(ib_work_request *ib_wr) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - - trios_declare_timer(callTime); - - struct ibv_mr *mr=NULL; - - uint32_t len; - - log_debug(nnti_debug_level, "enter"); - - len = sizeof(ib_wr->ack); - - if (config.use_memset) { - trios_start_timer(callTime); - memset(&ib_wr->ack, 0, len); - trios_stop_timer("memset", callTime); - } - if (config.use_mlock) { - trios_start_timer(callTime); - mlock(&ib_wr->ack, len); - trios_stop_timer("mlock", callTime); - } - - trios_start_timer(callTime); - mr = ibv_reg_mr_wrapper(transport_global_data.pd, &ib_wr->ack, len, (ibv_access_flags)(IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE)); - if (!mr) { - log_error(nnti_debug_level, "failed to register memory region"); - perror("errno"); - return errno; - } - trios_stop_timer("register", callTime); - - ib_wr->ack_mr=mr; - - log_debug(nnti_debug_level, "exit (mr==%p, addr %p, length %lu, lkey %x, rkey %x)...", mr, mr->addr, mr->length, mr->lkey, mr->rkey); - - return (rc); -} - -static int unregister_memory_segment(struct ibv_mr *mr) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - int ibv_rc=0; - trios_declare_timer(callTime); - - log_debug(nnti_debug_level, "enter"); - - trios_start_timer(callTime); - ibv_rc=ibv_dereg_mr_wrapper(mr); - if (ibv_rc != 0) { - log_error(nnti_debug_level, "deregistering the memory buffer failed"); - } - trios_stop_timer("deregister", callTime); - - if (config.use_mlock) { - trios_start_timer(callTime); - munlock(mr->addr, mr->length); - trios_stop_timer("munlock", callTime); - } - - log_debug(nnti_debug_level, "exit"); - - return (rc); -} - -static int unregister_ack(ib_work_request *ib_wr) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - int ibv_rc=0; - trios_declare_timer(callTime); - - log_debug(nnti_debug_level, "enter"); - - if (ib_wr->ack_mr!=NULL) { - trios_start_timer(callTime); - ibv_rc=ibv_dereg_mr_wrapper(ib_wr->ack_mr); - if (ibv_rc != 0) { - log_error(nnti_debug_level, "deregistering the ACK buffer failed"); - } - trios_stop_timer("deregister", callTime); - } - - if (config.use_mlock) { - trios_start_timer(callTime); - munlock(ib_wr->ack_mr->addr, ib_wr->ack_mr->length); - trios_stop_timer("munlock", callTime); - } - - log_debug(nnti_debug_level, "exit"); - - return (rc); -} - -static void send_ack ( - ib_work_request *ib_wr) -{ - struct ibv_send_wr *bad_wr; - - NNTI_buffer_t *reg_buf=NULL; - ib_memory_handle *ib_mem_hdl=NULL; - - reg_buf=ib_wr->reg_buf; - assert(reg_buf); - ib_mem_hdl=IB_MEM_HDL(reg_buf); - assert(ib_mem_hdl); - - log_debug(nnti_debug_level, "sending ACK to (ack_sge.addr=%p, ack_sge.length=%llu, ack_sq_wr.ib_wr.rdma.rkey=%x, ack_sq_wr.ib_wr.rdma.remote_addr=%p)", - (void *) ib_wr->ack_sge.addr, - (uint64_t)ib_wr->ack_sge.length, - ib_wr->ack_sq_wr.wr.rdma.rkey, - (void *) ib_wr->ack_sq_wr.wr.rdma.remote_addr); - - if (ibv_post_send_wrapper(ib_wr->qp, &ib_wr->ack_sq_wr, &bad_wr)) { - log_error(nnti_debug_level, "failed to post send: %s", strerror(errno)); - } - - return; -} - -static ib_work_request *decode_work_request( - const struct ibv_wc *wc) -{ - ib_connection *conn=NULL; - ib_work_request *ib_wr=NULL; - NNTI_buffer_t *event_buf=NULL; - ib_memory_handle *ib_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - if (wc->opcode==IBV_WC_RECV) { - log_debug(nnti_debug_level, "wc->opcode is IBV_WC_RECV, so this is a new request. wc.wr_id is a key to the work request map."); - -// ib_mem_hdl=IB_MEM_HDL(transport_global_data.req_queue.reg_buf); -// assert(ib_mem_hdl); - - nthread_lock(&nnti_wrmap_lock); - if (wrmap.find(wc->wr_id) != wrmap.end()) { - ib_wr = wrmap[wc->wr_id]; - } - nthread_unlock(&nnti_wrmap_lock); - assert(ib_wr); - - } else if (wc->opcode==IBV_WC_SEND) { - log_debug(nnti_debug_level, "wc->opcode is IBV_WC_SEND, so this is a send request (IB_OP_SEND_REQUEST). wc.wr_id is a key to the work request map."); - - nthread_lock(&nnti_wrmap_lock); - if (wrmap.find(wc->wr_id) != wrmap.end()) { - ib_wr = wrmap[wc->wr_id]; - } - nthread_unlock(&nnti_wrmap_lock); - assert(ib_wr); - - } else { - if (wc->imm_data == 0) { - log_debug(nnti_debug_level, "wc->imm_data == 0, so I am the initiator. wc.wr_id is a key to the work request map."); - - // This is not a request buffer and I am the initiator, so wc.wr_id is the hash of the work request - nthread_lock(&nnti_wrmap_lock); - if (wrmap.find(wc->wr_id) != wrmap.end()) { - ib_wr = wrmap[wc->wr_id]; - } - nthread_unlock(&nnti_wrmap_lock); - assert(ib_wr); - - } else { - log_debug(nnti_debug_level, "wc->imm_data != 0, so I am the target. wc->imm_data is either the hash of a buffer or the hash of an ib_wr."); - - // This is not a request buffer and I am the target, so wc.imm_data is the hash of either the buffer or the work request - event_buf = get_buf_bufhash(wc->imm_data); - assert(event_buf); - ib_mem_hdl=IB_MEM_HDL(event_buf); - assert(ib_mem_hdl); - ib_wr = ib_mem_hdl->wr_queue.front(); - assert(ib_wr); - } - } - - log_debug(nnti_debug_level, "exit (ib_wr==%p)", ib_wr); - - return(ib_wr); -} - -static int cancel_wr( - ib_work_request *ib_wr) -{ - NNTI_result_t rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter (ib_wr=%p)", ib_wr); - - ib_wr->state=NNTI_IB_WR_STATE_WAIT_COMPLETE; - ib_wr->nnti_wr->result=NNTI_ECANCELED; - - log_debug(nnti_debug_level, "exit (ib_wr==%p)", ib_wr); - - return(rc); -} - -static int process_event( - ib_work_request *ib_wr, - const struct ibv_wc *wc) -{ - NNTI_result_t rc=NNTI_OK; - - log_level debug_level=nnti_debug_level; - - log_debug(nnti_debug_level, "enter (ib_wr=%p)", ib_wr); - - if ((ib_wr->nnti_wr) && (ib_wr->nnti_wr->ops == NNTI_BOP_ATOMICS)) { - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - ib_wr->nnti_wr->result=NNTI_OK; - return NNTI_OK; - } - - if (wc->status == IBV_WC_RNR_RETRY_EXC_ERR) { - ib_wr->state =NNTI_IB_WR_STATE_WAIT_COMPLETE; - ib_wr->nnti_wr->result=NNTI_EDROPPED; - return NNTI_EDROPPED; - } else if (wc->status != IBV_WC_SUCCESS) { - ib_wr->state =NNTI_IB_WR_STATE_WAIT_COMPLETE; - ib_wr->nnti_wr->result=NNTI_EIO; - return NNTI_EIO; - } - - log_debug(nnti_debug_level, "ib_wr=%p; ib_wr->last_op=%d", ib_wr, ib_wr->last_op); - - ib_wr->last_wc = *wc; - - if (ib_wr->last_op == IB_OP_SEND_REQUEST) { - if (wc->opcode==IBV_WC_SEND) { - log_debug(debug_level, "send completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - ib_wr->nnti_wr->result=NNTI_OK; - } - } else if (ib_wr->last_op == IB_OP_SEND_BUFFER) { - if (wc->opcode==IBV_WC_RDMA_WRITE) { - log_debug(debug_level, "send completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - ib_wr->nnti_wr->result=NNTI_OK; - } - } else if (ib_wr->last_op == IB_OP_PUT_INITIATOR) { - log_debug(debug_level, "RDMA write event - wc==%p, ib_wr==%p, state==%d", wc, ib_wr, ib_wr->state); - if (wc->opcode==IBV_WC_RDMA_WRITE) { - if (wc->wc_flags==0) { - if (ib_wr->state==NNTI_IB_WR_STATE_STARTED) { - log_debug(debug_level, "RDMA write (initiator) completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->sq_wr_completed_count++; - log_debug(nnti_debug_level, "ib_wr->sq_wr_completed_count=%d", ib_wr->sq_wr_completed_count); - } - if (ib_wr->sq_wr_completed_count==ib_wr->sq_wr_count) { - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - if (!config.use_rdma_target_ack) { - ib_wr->nnti_wr->result=NNTI_OK; - } - } - } - if (wc->wc_flags==IBV_WC_WITH_IMM) { - if ((config.use_rdma_target_ack) && - (ib_wr->state == NNTI_IB_WR_STATE_RDMA_COMPLETE)) { - log_debug(debug_level, "RDMA write ACK (initiator) completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_ACK_COMPLETE; - ib_wr->nnti_wr->result=NNTI_OK; - } - } - } -// if (ib_wr->state == NNTI_IB_WR_STATE_RDMA_COMPLETE) { -// print_xfer_buf((void *)ib_wr->reg_buf->payload, ib_wr->reg_buf->payload_size); -// print_ack_buf(&ib_wr->ack); -// } - } else if (ib_wr->last_op == IB_OP_GET_INITIATOR) { - log_debug(debug_level, "RDMA read event - wc==%p, ib_wr==%p, state==%d", wc, ib_wr, ib_wr->state); - if ((wc->opcode==IBV_WC_RDMA_READ) && - (wc->wc_flags==0)) { - if (ib_wr->state==NNTI_IB_WR_STATE_STARTED) { - log_debug(debug_level, "RDMA read (initiator) completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->sq_wr_completed_count++; - log_debug(nnti_debug_level, "ib_wr->sq_wr_completed_count=%d", ib_wr->sq_wr_completed_count); - } - if (ib_wr->sq_wr_completed_count==ib_wr->sq_wr_count) { - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - if (!config.use_rdma_target_ack) { - ib_wr->nnti_wr->result=NNTI_OK; - } - } - } - else if ((config.use_rdma_target_ack) && - ((wc->opcode==IBV_WC_RDMA_WRITE) && - (wc->wc_flags==IBV_WC_WITH_IMM))) { - if (ib_wr->state == NNTI_IB_WR_STATE_RDMA_COMPLETE) { - log_debug(debug_level, "RDMA read ACK (initiator) completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_ACK_COMPLETE; - ib_wr->nnti_wr->result=NNTI_OK; - } - } -// if (ib_wr->state == NNTI_IB_WR_STATE_RDMA_COMPLETE) { -// print_xfer_buf((void *)ib_wr->reg_buf->payload, ib_wr->reg_buf->payload_size); -// print_ack_buf(&ib_wr->ack); -// } - } else if (ib_wr->last_op == IB_OP_NEW_REQUEST) { - if (wc->opcode==IBV_WC_RECV) { - log_debug(debug_level, "recv completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - -// if (transport_global_data.req_queue.req_received == transport_global_data.srq_count) { -// log_debug(debug_level, "resetting req_queue.req_received to 0"); -// transport_global_data.req_queue.req_received=0; -// } -// if (transport_global_data.req_queue.req_received != (ib_wr->offset/ib_wr->length)) { -// log_warn(debug_level, "req_queue.req_received(%llu) != (ib_wr->offset(%llu)/ib_wr->length(%llu))", -// transport_global_data.req_queue.req_received, ib_wr->offset, ib_wr->length); -// } - transport_global_data.req_queue.req_received++; - - if (ib_wr->cq == transport_global_data.req_cq) { - transport_global_data.req_srq_count--; - log_debug(nnti_debug_level, "transport_global_data.req_srq_count==%ld", transport_global_data.req_srq_count); - } - } - } else if (ib_wr->last_op == IB_OP_RECEIVE) { - if (wc->opcode==IBV_WC_RECV_RDMA_WITH_IMM) { - log_debug(debug_level, "recv completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - - if (ib_wr->cq == transport_global_data.data_cq) { - transport_global_data.data_srq_count--; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - } - } else if (ib_wr->last_op == IB_OP_PUT_TARGET) { - if (wc->opcode==IBV_WC_RECV_RDMA_WITH_IMM) { - log_debug(debug_level, "RDMA write (target) completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - - if (ib_wr->cq == transport_global_data.data_cq) { - transport_global_data.data_srq_count--; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - } -// if (ib_wr->op_state == RDMA_WRITE_COMPLETE) { -// print_xfer_buf((void *)ib_wr->reg_buf->payload, ib_wr->reg_buf->payload_size); -// print_ack_buf(&ib_wr->ack); -// } - } else if (ib_wr->last_op == IB_OP_GET_TARGET) { - if (wc->opcode==IBV_WC_RECV_RDMA_WITH_IMM) { - log_debug(debug_level, "RDMA read (target) completion - wc==%p, ib_wr==%p", wc, ib_wr); - ib_wr->state=NNTI_IB_WR_STATE_RDMA_COMPLETE; - - if (ib_wr->cq == transport_global_data.data_cq) { - transport_global_data.data_srq_count--; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - } -// if (ib_wr->op_state == RDMA_READ_COMPLETE) { -// print_xfer_buf((void *)ib_wr->reg_buf->payload, ib_wr->reg_buf->payload_size); -// print_ack_buf(&ib_wr->ack); -// } - } else { - log_error(nnti_debug_level, "unknown ib_wr->last_op(%d)", ib_wr->last_op); - } - - return (rc); -} - -static NNTI_result_t post_recv_work_request( - NNTI_buffer_t *reg_buf, - int64_t wr_id, - uint64_t offset) -{ - int ibv_rc=0; - - struct ibv_recv_wr *bad_wr=NULL; - - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - struct ibv_srq *srq=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - ib_mem_hdl=IB_MEM_HDL(reg_buf); - assert(ib_mem_hdl); - - if (reg_buf->ops==NNTI_BOP_RECV_QUEUE) { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - nthread_lock_init(&ib_wr->lock); - } else { - if (config.use_wr_pool) { - ib_wr=wr_pool_sendrecv_pop(); - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - nthread_lock_init(&ib_wr->lock); - } - } - assert(ib_wr); - log_debug(nnti_debug_level, "allocated ib_wr (ib_wr=%p)", ib_wr); - ib_wr->reg_buf = reg_buf; - ib_wr->offset = offset; - ib_wr->length = reg_buf->payload_size; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->state=NNTI_IB_WR_STATE_POSTED; - - if (reg_buf->ops==NNTI_BOP_RECV_QUEUE) { - ib_wr->last_op=IB_OP_NEW_REQUEST; - - srq =transport_global_data.req_srq; - ib_wr->comp_channel=transport_global_data.req_comp_channel; - ib_wr->cq =transport_global_data.req_cq; - } else { - ib_wr->last_op=IB_OP_RECEIVE; - - srq =transport_global_data.data_srq; - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - } - - ib_wr->sge_count=reg_buf->buffer_segments.NNTI_remote_addr_array_t_len; - - if (ib_wr->sge_count == 1) { - - ib_wr->sge_list=&ib_wr->sge; - - ib_wr->sge_list[0].addr =(uint64_t)ib_mem_hdl->mr_list[0]->addr+offset; - ib_wr->sge_list[0].length=reg_buf->payload_size; - ib_wr->sge_list[0].lkey =ib_mem_hdl->mr_list[0]->lkey; - - } else { - - ib_wr->sge_list=(struct ibv_sge *)calloc(ib_wr->sge_count, sizeof(struct ibv_sge)); - - for (int i=0;isge_count;i++) { - ib_wr->sge_list[i].addr =(uint64_t)ib_mem_hdl->mr_list[i]->addr; - ib_wr->sge_list[i].length=ib_mem_hdl->mr_list[i]->length; - ib_wr->sge_list[i].lkey =ib_mem_hdl->mr_list[i]->lkey; - } - - } - - if (wr_id == -1) { - ib_wr->rq_wr.wr_id=(uint64_t)ib_wr->key; - } else { - ib_wr->rq_wr.wr_id=wr_id; - } - ib_wr->rq_wr.sg_list=ib_wr->sge_list; - ib_wr->rq_wr.num_sge=ib_wr->sge_count; - - log_debug(nnti_debug_level, "sge_count=%d, sge_list=%p", ib_wr->sge_count, ib_wr->sge_list); - - if ((ib_wr->cq == transport_global_data.data_cq) && (transport_global_data.data_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.data_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - if ((ib_wr->cq == transport_global_data.req_cq) && (transport_global_data.req_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.req_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.req_srq_count==%ld", transport_global_data.req_srq_count); - } - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - nthread_lock(&ib_mem_hdl->wr_queue_lock); - ib_mem_hdl->wr_queue.push_back(ib_wr); - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static NNTI_result_t post_ack_recv_work_request( - NNTI_buffer_t *reg_buf) -{ - int ibv_rc=0; - - struct ibv_recv_wr *bad_wr=NULL; - struct ibv_srq *srq; - - ib_memory_handle *ib_mem_hdl=NULL; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - ib_mem_hdl=IB_MEM_HDL(reg_buf); - assert(ib_mem_hdl); - - if (config.use_wr_pool) { - ib_wr=wr_pool_rdma_pop(); - } else { - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - nthread_lock_init(&ib_wr->lock); - } - assert(ib_wr); - log_debug(nnti_debug_level, "allocated ib_wr (ib_wr=%p)", ib_wr); - ib_wr->reg_buf = reg_buf; - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - - ib_wr->state=NNTI_IB_WR_STATE_POSTED; - - if (reg_buf->ops==NNTI_BOP_RECV_QUEUE) { - ib_wr->last_op=IB_OP_NEW_REQUEST; - - srq =transport_global_data.req_srq; - ib_wr->comp_channel=transport_global_data.req_comp_channel; - ib_wr->cq =transport_global_data.req_cq; - } else { - ib_wr->last_op=IB_OP_RECEIVE; - - srq =transport_global_data.data_srq; - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - } - - if (!config.use_wr_pool) { - register_ack(ib_wr); - } - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_size = ib_wr->ack_mr->length; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_buf = (uint64_t)ib_wr->ack_mr->addr; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_key = ib_wr->ack_mr->rkey; - - ib_wr->sge.addr =(uint64_t)ib_wr->ack_mr->addr; - ib_wr->sge.length=ib_wr->ack_mr->length; - ib_wr->sge.lkey =ib_wr->ack_mr->lkey; - - ib_wr->rq_wr.wr_id =(uint64_t)ib_wr->key; - ib_wr->rq_wr.sg_list=&ib_wr->sge; - ib_wr->rq_wr.num_sge=1; - - if ((ib_wr->cq == transport_global_data.data_cq) && (transport_global_data.data_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.data_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - if ((ib_wr->cq == transport_global_data.req_cq) && (transport_global_data.req_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.req_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.req_srq_count==%ld", transport_global_data.req_srq_count); - } - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - nthread_lock(&ib_mem_hdl->wr_queue_lock); - ib_mem_hdl->wr_queue.push_back(ib_wr); - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - nthread_lock(&nnti_wrmap_lock); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static NNTI_result_t repost_recv_work_request( - NNTI_work_request_t *wr, - ib_work_request *ib_wr) -{ - int ibv_rc=0; - - struct ibv_recv_wr *bad_wr=NULL; - - ib_memory_handle *ib_mem_hdl=NULL; - - struct ibv_srq *srq=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p ; ib_wr=%p)", wr, ib_wr); - - assert(wr); - assert(ib_wr); - ib_mem_hdl=IB_MEM_HDL(wr->reg_buf); - assert(ib_mem_hdl); - - nthread_lock(&nnti_wrmap_lock); - wrmap_iter_t m_victim=wrmap.find(ib_wr->key); - if (m_victim != wrmap.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p (key=%lx) from the wrmap", ib_wr, ib_wr->key); - wrmap.erase(m_victim); - } - - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - ib_wr->rq_wr.wr_id=(uint64_t)ib_wr->key; - - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - ib_wr->state=NNTI_IB_WR_STATE_POSTED; - - if (wr->reg_buf->ops==NNTI_BOP_RECV_QUEUE) { - ib_wr->last_op=IB_OP_NEW_REQUEST; - - srq =transport_global_data.req_srq; - ib_wr->comp_channel=transport_global_data.req_comp_channel; - ib_wr->cq =transport_global_data.req_cq; - } else { - ib_wr->last_op=IB_OP_RECEIVE; - - srq =transport_global_data.data_srq; - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - } - - if ((ib_wr->cq == transport_global_data.data_cq) && (transport_global_data.data_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.data_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - if ((ib_wr->cq == transport_global_data.req_cq) && (transport_global_data.req_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.req_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.req_srq_count==%ld", transport_global_data.req_srq_count); - } - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - -static NNTI_result_t repost_ack_recv_work_request( - NNTI_work_request_t *wr, - ib_work_request *ib_wr) -{ - int ibv_rc=0; - - struct ibv_recv_wr *bad_wr=NULL; - struct ibv_srq *srq; - - ib_memory_handle *ib_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - ib_wr=IB_WORK_REQUEST(wr); - assert(ib_wr); - ib_mem_hdl=IB_MEM_HDL(wr->reg_buf); - assert(ib_mem_hdl); - - ib_wr->state=NNTI_IB_WR_STATE_POSTED; - - if (wr->reg_buf->ops==NNTI_BOP_RECV_QUEUE) { - ib_wr->last_op=IB_OP_NEW_REQUEST; - - srq =transport_global_data.req_srq; - ib_wr->comp_channel=transport_global_data.req_comp_channel; - ib_wr->cq =transport_global_data.req_cq; - } else { - ib_wr->last_op=IB_OP_RECEIVE; - - srq =transport_global_data.data_srq; - ib_wr->comp_channel=transport_global_data.data_comp_channel; - ib_wr->cq =transport_global_data.data_cq; - } - - if ((ib_wr->cq == transport_global_data.data_cq) && (transport_global_data.data_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.data_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.data_srq_count==%ld", transport_global_data.data_srq_count); - } - if ((ib_wr->cq == transport_global_data.req_cq) && (transport_global_data.req_srq_count < (transport_global_data.srq_count/2))) { - log_debug(nnti_debug_level, "posting ib_wr=%p (key=%lx)", ib_wr, ib_wr->key); - ibv_rc=ibv_post_srq_recv_wrapper(srq, &ib_wr->rq_wr, &bad_wr); - if (ibv_rc) { - log_error(nnti_debug_level, "failed to post SRQ recv (rq_wr=%p ; bad_wr=%p): %s", - &ib_wr->rq_wr, bad_wr, strerror(ibv_rc)); - return (NNTI_result_t)errno; - } - - transport_global_data.req_srq_count++; - log_debug(nnti_debug_level, "transport_global_data.req_srq_count==%ld", transport_global_data.req_srq_count); - } - - log_debug(nnti_debug_level, "pushing ib_wr=%p", ib_wr); - nthread_lock(&ib_mem_hdl->wr_queue_lock); - ib_mem_hdl->wr_queue.push_back(ib_wr); - nthread_unlock(&ib_mem_hdl->wr_queue_lock); - - nthread_lock(&nnti_wrmap_lock); - wrmap_iter_t m_victim=wrmap.find(ib_wr->key); - if (m_victim != wrmap.end()) { - log_debug(nnti_debug_level, "erasing ib_wr=%p (key=%lx) from the wrmap", ib_wr, ib_wr->key); - wrmap.erase(m_victim); - } - ib_wr->key = nthread_counter_increment(&nnti_wrmap_counter); - log_debug(nnti_debug_level, "wrmap[key(%lx)]=ib_wr(%p)", ib_wr->key, ib_wr); - assert(wrmap.find(ib_wr->key) == wrmap.end()); - wrmap[ib_wr->key] = ib_wr; - nthread_unlock(&nnti_wrmap_lock); - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - -static int8_t is_wr_canceling( - ib_work_request *ib_wr) -{ - int8_t rc=FALSE; - - nthread_lock(&ib_wr->lock); - if (ib_wr->state == NNTI_IB_WR_STATE_CANCELING) { - rc=TRUE; - } - nthread_unlock(&ib_wr->lock); - - log_debug(nnti_debug_level, "exit (rc=%d)", rc); - - return(rc); -} - -static int8_t is_wr_complete( - ib_work_request *ib_wr) -{ - int8_t rc=FALSE; - - nthread_lock(&ib_wr->lock); - if ((ib_wr->state==NNTI_IB_WR_STATE_RDMA_COMPLETE) || - (ib_wr->state==NNTI_IB_WR_STATE_WAIT_COMPLETE)) { - rc=TRUE; - } - nthread_unlock(&ib_wr->lock); - - log_debug(nnti_debug_level, "exit (rc=%d)", rc); - - return(rc); -} - -static int8_t is_any_wr_complete( - ib_work_request **wr_list, - const uint32_t wr_count, - uint32_t *which) -{ - int8_t rc=FALSE; - - log_debug(nnti_debug_level, "enter"); - - for (uint32_t i=0;iop = wr->ops; - if (is_wr_complete(ib_wr)) { - status->result = wr->result; - } else { - status->result = nnti_rc; - } - if (status->result==NNTI_OK) { - -// print_ack_buf(&ib_wr->ack); -// print_wr(ib_wr); - - conn = get_conn_qpn(ib_wr->last_wc.qp_num); - - if (ib_wr->nnti_wr->ops != NNTI_BOP_ATOMICS) { - status->start = (uint64_t)ib_wr->reg_buf->payload; - } - switch (ib_wr->last_op) { - case IB_OP_PUT_INITIATOR: - case IB_OP_SEND_REQUEST: - case IB_OP_SEND_BUFFER: - copy_peer( - &status->src, - &transport_global_data.me); - copy_peer( - &status->dest, - &conn->peer); - break; - case IB_OP_GET_TARGET: - if (config.use_rdma_target_ack) { - copy_peer( - &status->src, - &transport_global_data.me); - copy_peer( - &status->dest, - &conn->peer); - } - break; - case IB_OP_GET_INITIATOR: - case IB_OP_NEW_REQUEST: - case IB_OP_RECEIVE: - copy_peer( - &status->src, - &conn->peer); - copy_peer( - &status->dest, - &transport_global_data.me); - break; - case IB_OP_PUT_TARGET: - if (config.use_rdma_target_ack) { - copy_peer( - &status->src, - &conn->peer); - copy_peer( - &status->dest, - &transport_global_data.me); - } - break; - } - switch (ib_wr->last_op) { - case IB_OP_NEW_REQUEST: - status->offset = ib_wr->offset; - status->length = ib_wr->last_wc.byte_len; - break; - case IB_OP_SEND_REQUEST: - case IB_OP_SEND_BUFFER: - case IB_OP_RECEIVE: - status->offset = 0; - status->length = ib_wr->last_wc.byte_len; - break; - case IB_OP_GET_INITIATOR: - case IB_OP_PUT_INITIATOR: - status->offset = ib_wr->offset; - status->length = ib_wr->length; - break; - case IB_OP_GET_TARGET: - case IB_OP_PUT_TARGET: - if (config.use_rdma_target_ack) { - status->offset = ib_wr->ack.offset; - status->length = ib_wr->ack.length; - } - break; - } - } - - trios_stop_timer("create_status - total", total_time); -} - -static void create_peer(NNTI_peer_t *peer, char *name, NNTI_ip_addr addr, NNTI_tcp_port port) -{ - log_debug(nnti_debug_level, "enter"); - - sprintf(peer->url, "ib://%s:%u/", name, ntohs(port)); - - peer->peer.transport_id =NNTI_TRANSPORT_IB; - peer->peer.NNTI_remote_process_t_u.ib.addr=addr; - peer->peer.NNTI_remote_process_t_u.ib.port=port; - - log_debug(nnti_debug_level, "exit"); -} - -static void copy_peer(NNTI_peer_t *dest, NNTI_peer_t *src) -{ - log_debug(nnti_debug_level, "enter"); - - strcpy(dest->url, src->url); - - dest->peer.transport_id =NNTI_TRANSPORT_IB; - dest->peer.NNTI_remote_process_t_u.ib.addr=src->peer.NNTI_remote_process_t_u.ib.addr; - dest->peer.NNTI_remote_process_t_u.ib.port=src->peer.NNTI_remote_process_t_u.ib.port; - - log_debug(nnti_debug_level, "exit"); -} - -static int init_server_listen_socket() -{ - NNTI_result_t rc=NNTI_OK; - int flags; - struct hostent *host_entry; - struct sockaddr_in skin; - socklen_t skin_size=sizeof(struct sockaddr_in); - - transport_global_data.listen_sock = socket(AF_INET, SOCK_STREAM, 0); - if (transport_global_data.listen_sock < 0) - log_error(nnti_debug_level, "failed to create tcp socket: %s", strerror(errno)); - - flags = 1; - if (setsockopt(transport_global_data.listen_sock, SOL_SOCKET, SO_REUSEADDR, &flags, sizeof(flags)) < 0) - log_error(nnti_debug_level, "failed to set tcp socket REUSEADDR flag: %s", strerror(errno)); - - flags=fcntl(transport_global_data.listen_sock, F_GETFL, 0); - fcntl(transport_global_data.listen_sock, F_SETFL, flags | O_NONBLOCK); - - if (transport_global_data.listen_name[0]!='\0') { - log_debug(nnti_debug_level, "using hostname from command-line (%s).", transport_global_data.listen_name); - } else { - gethostname(transport_global_data.listen_name, NNTI_HOSTNAME_LEN); - log_debug(nnti_debug_level, "hostname not given on command-line. using gethostname() result (%s).", transport_global_data.listen_name); - } - /* lookup the host provided on the command line */ - host_entry = gethostbyname(transport_global_data.listen_name); - if (!host_entry) { - log_warn(nnti_debug_level, "failed to resolve server name (%s): %s", transport_global_data.listen_name, strerror(errno)); - return NNTI_ENOENT; - } - - memset(&skin, 0, sizeof(skin)); - skin.sin_family = AF_INET; - memcpy(&skin.sin_addr, host_entry->h_addr_list[0], (size_t) host_entry->h_length); - /* 0 here means to bind to a random port assigned by the kernel */ - skin.sin_port = 0; - -retry: - if (bind(transport_global_data.listen_sock, (struct sockaddr *) &skin, sizeof(skin)) < 0) { - if (errno == EINTR) { - goto retry; - } else { - log_error(nnti_debug_level, "failed to bind tcp socket: %s", strerror(errno)); - } - } - /* after the bind, get the "name" for the socket. the "name" contains the port assigned by the kernel. */ - getsockname(transport_global_data.listen_sock, (struct sockaddr *)&skin, &skin_size); - transport_global_data.listen_addr = (uint32_t)skin.sin_addr.s_addr; - transport_global_data.listen_port = (uint16_t)skin.sin_port; - log_debug(nnti_debug_level, "listening on ip(%s) addr(%u) port(%u)", - transport_global_data.listen_name, - (unsigned int)ntohl(skin.sin_addr.s_addr), - (unsigned int)ntohs(skin.sin_port)); - if (listen(transport_global_data.listen_sock, 1024) < 0) - log_error(nnti_debug_level, "failed to listen on tcp socket: %s", strerror(errno)); - - return rc; -} - - -static void transition_connection_to_ready( - int sock, - ib_connection *conn) -{ - int rc=NNTI_OK; - int min_rnr_timer; - int ack_timeout; - int retry_count; - trios_declare_timer(callTime); - - /* bring the two QPs up to RTR */ - trios_start_timer(callTime); - - min_rnr_timer=1; /* means 0.01ms delay before sending RNR NAK */ - ack_timeout =17; /* time to wait for ACK/NAK before retransmitting. 4.096us * 2^17 == 0.536s */ - if (config.drop_if_full_queue) { - retry_count=1; /* number of retries if no answer on primary path or if remote sends RNR NAK */ - } else { - retry_count=7; /* number of retries if no answer on primary path or if remote sends RNR NAK. 7 has special meaning of infinite retries. */ - } - transition_qp_from_reset_to_ready(conn->req_qp.qp, conn->peer_req_qpn, conn->peer_lid, min_rnr_timer, ack_timeout, retry_count); - - - min_rnr_timer=31; /* means 491.52ms delay before sending RNR NAK */ - ack_timeout =17; /* time to wait for ACK/NAK before retransmitting. 4.096us * 2^17 == 0.536s */ - retry_count =7; /* number of retries if no answer on primary path or if remote sends RNR NAK. 7 has special meaning of infinite retries. */ - transition_qp_from_reset_to_ready(conn->data_qp.qp, conn->data_qp.peer_qpn, conn->peer_lid, min_rnr_timer, ack_timeout, retry_count); - - trios_stop_timer("transition_qp_from_reset_to_ready", callTime); - - trios_start_timer(callTime); - /* final sychronization to ensure both sides have posted RTRs */ - rc = tcp_exchange(sock, 0, &rc, &rc, sizeof(rc)); - trios_stop_timer("exch data", callTime); -} - -static void transition_qp_from_reset_to_ready( - struct ibv_qp *qp, - uint32_t peer_qpn, - int peer_lid, - int min_rnr_timer, - int ack_timeout, - int retry_count) -{ - enum ibv_qp_attr_mask mask; - struct ibv_qp_attr attr; - - log_debug(nnti_debug_level, "enter (qp=%p ; qp->qp_num=%u ; peer_qpn=%u ; peer_lid=%u)", qp, qp->qp_num, peer_qpn, peer_lid); - - /* Transition QP to Init */ - mask = (enum ibv_qp_attr_mask) - ( IBV_QP_STATE - | IBV_QP_ACCESS_FLAGS - | IBV_QP_PKEY_INDEX - | IBV_QP_PORT ); - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_INIT; - attr.qp_access_flags = IBV_ACCESS_LOCAL_WRITE | - IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | - IBV_ACCESS_REMOTE_ATOMIC; - attr.pkey_index = 0; - attr.port_num = transport_global_data.nic_port; - if (ibv_modify_qp_wrapper(qp, &attr, mask)) { - log_error(nnti_debug_level, "failed to modify qp from RESET to INIT state"); - } - - /* Transition QP to Ready-to-Receive (RTR) */ - mask = (enum ibv_qp_attr_mask) - ( IBV_QP_STATE - | IBV_QP_MAX_DEST_RD_ATOMIC - | IBV_QP_AV - | IBV_QP_PATH_MTU - | IBV_QP_RQ_PSN - | IBV_QP_DEST_QPN - | IBV_QP_MIN_RNR_TIMER ); - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RTR; - attr.max_dest_rd_atomic = 1; - attr.ah_attr.dlid = peer_lid; - attr.ah_attr.port_num = transport_global_data.nic_port; - attr.path_mtu = IBV_MTU_1024; - attr.rq_psn = 0; - attr.dest_qp_num = peer_qpn; - attr.min_rnr_timer = min_rnr_timer; /* delay before sending RNR NAK */ - if (ibv_modify_qp_wrapper(qp, &attr, mask)) { - log_error(nnti_debug_level, "failed to modify qp from INIT to RTR state"); - } - - /* Transition QP to Ready-to-Send (RTS) */ - mask = (enum ibv_qp_attr_mask) - ( IBV_QP_STATE - | IBV_QP_SQ_PSN - | IBV_QP_MAX_QP_RD_ATOMIC - | IBV_QP_TIMEOUT - | IBV_QP_RETRY_CNT - | IBV_QP_RNR_RETRY ); - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RTS; - attr.sq_psn = 0; - attr.max_rd_atomic = 1; - attr.timeout = ack_timeout; /* time to wait for ACK/NAK before retransmitting. 4.096us * 2^ack_timeout */ - attr.retry_cnt = retry_count; /* number of retries if no answer on primary path */ - attr.rnr_retry = retry_count; /* number of retries if remote sends RNR NAK */ - if (ibv_modify_qp_wrapper(qp, &attr, mask)) { - log_error(nnti_debug_level, "failed to modify qp from RTR to RTS state"); - } - - log_debug(nnti_debug_level, "exit"); -} - -static void transition_qp_from_error_to_ready( - struct ibv_qp *qp, - uint32_t peer_qpn, - int peer_lid, - int min_rnr_timer, - int ack_timeout, - int retry_count) -{ - enum ibv_qp_attr_mask mask; - struct ibv_qp_attr attr; - - log_debug(nnti_debug_level, "enter (qp=%p ; qp->qp_num=%u ; peer_qpn=%u ; peer_lid=%u)", qp, qp->qp_num, peer_qpn, peer_lid); - - /* Transition QP to Reset */ - mask = (enum ibv_qp_attr_mask) - ( IBV_QP_STATE ); - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_RESET; - if (ibv_modify_qp_wrapper(qp, &attr, mask)) { - log_error(nnti_debug_level, "failed to modify qp from ERROR to RESET state"); - } - - transition_qp_from_reset_to_ready(qp, peer_qpn, peer_lid, min_rnr_timer, ack_timeout, retry_count); - - log_debug(nnti_debug_level, "exit"); -} - -static void transition_connection_to_error( - ib_connection *conn) -{ -// int i; - trios_declare_timer(callTime); - - /* bring the two QPs up to RTR */ - trios_start_timer(callTime); - transition_qp_to_error(conn->req_qp.qp, conn->peer_req_qpn, conn->peer_lid); - transition_qp_to_error(conn->data_qp.qp, conn->data_qp.peer_qpn, conn->peer_lid); - trios_stop_timer("transition_qp_to_error", callTime); -} - -static void transition_qp_to_error( - struct ibv_qp *qp, - uint32_t peer_qpn, - int peer_lid) -{ - struct ibv_qp_attr attr; - - /* Transition QP to Error */ - memset(&attr, 0, sizeof(attr)); - attr.qp_state = IBV_QPS_ERR; - if (ibv_modify_qp_wrapper(qp, &attr, IBV_QP_STATE)) { - log_error(nnti_debug_level, "failed to modify qp to ERROR state"); - } -} - - -/* - * Try hard to read the whole buffer. Abort on read error. - */ -static int tcp_read(int sock, void *incoming, size_t len) -{ - int bytes_this_read=0; - int bytes_left=len; - int bytes_read=0; - - while (bytes_left > 0) { - bytes_this_read = read(sock, (char *)incoming + bytes_read, bytes_left); - if (bytes_this_read < 0) { - return bytes_this_read; - } - if (bytes_this_read == 0) { - break; - } - bytes_left -= bytes_this_read; - bytes_read += bytes_this_read; - } - return bytes_read; -} - -/* - * Try hard to write the whole buffer. Abort on write error. - */ -static int tcp_write(int sock, const void *outgoing, size_t len) -{ - int bytes_this_write=0; - int bytes_left=len; - int bytes_written=0; - - while (bytes_left > 0) { - bytes_this_write = write(sock, (const char *)outgoing + bytes_written, bytes_left); - if (bytes_this_write < 0) { - return bytes_this_write; - } - bytes_left -= bytes_this_write; - bytes_written += bytes_this_write; - } - return bytes_written; -} - -/* - * Two processes exchange data over a TCP socket. Both sides send and receive the - * same amount of data. Only one process can declare itself the server (is_server!=0), - * otherwise this will hang because both will wait for the read to complete. - * - * Server receives, then sends. - * Client sends, then receives. - */ -static int tcp_exchange(int sock, int is_server, void *incoming, void *outgoing, size_t len) -{ - int rc; - - if (is_server) { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_read(sock, incoming, len); - trios_stop_timer("tcp_read", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "server failed to read IB connection info: errno=%d", errno); - goto out; - } - if (rc != (int) len) { - log_error(nnti_debug_level, "partial read, %d/%d bytes", rc, (int) len); - rc = 1; - goto out; - } - } else { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_write(sock, outgoing, len); - trios_stop_timer("tcp_write", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "client failed to write IB connection info: errno=%d", errno); - goto out; - } - } - - if (is_server) { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_write(sock, outgoing, len); - trios_stop_timer("tcp_write", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "server failed to write IB connection info: errno=%d", errno); - goto out; - } - } else { - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = tcp_read(sock, incoming, len); - trios_stop_timer("tcp_read", callTime); - if (rc < 0) { - log_warn(nnti_debug_level, "client failed to read IB connection info: errno=%d", errno); - goto out; - } - if (rc != (int) len) { - log_error(nnti_debug_level, "partial read, %d/%d bytes", rc, (int) len); - rc = 1; - goto out; - } - } - - rc = 0; - -out: - return rc; -} - -static int new_client_connection( - ib_connection *c, - int sock) -{ - int rc; -// int flags=0; - struct ibv_qp_init_attr att; - - /* - * IB parameters passed through TCP to establish the IB connection. - */ - struct { - char name[NNTI_HOSTNAME_LEN]; - uint32_t addr; - uint32_t port; - uint32_t lid; - uint32_t req_qpn; - uint32_t my_qpn; - uint32_t peer_qpn; - uint32_t atomics_rkey; - uint64_t atomics_addr; - } param_in, param_out; - - trios_declare_timer(callTime); - - // Initialize to avoid an "uninitialized bytes messages from valgrind - memset(¶m_out, 0, sizeof(param_out)); - memset(¶m_in, 0, sizeof(param_in)); - - strcpy(param_out.name, transport_global_data.listen_name); - param_out.addr = htonl((uint32_t)transport_global_data.listen_addr); - param_out.port = htonl((uint32_t)transport_global_data.listen_port); - - /* create the main queue pair */ - memset(&att, 0, sizeof(att)); - att.qp_context = c; - att.send_cq = transport_global_data.req_cq; - att.recv_cq = transport_global_data.req_cq; - att.srq = transport_global_data.req_srq; - att.cap.max_recv_wr = transport_global_data.qp_count; - att.cap.max_send_wr = transport_global_data.qp_count; - att.cap.max_recv_sge = 32; - att.cap.max_send_sge = 32; - att.qp_type = IBV_QPT_RC; - - trios_start_timer(callTime); - c->req_qp.qp = ibv_create_qp_wrapper(transport_global_data.pd, &att); - if (!c->req_qp.qp) { - log_error(nnti_debug_level, "failed to create QP: %s", strerror(errno)); - } - c->req_qp.qpn = c->req_qp.qp->qp_num; - trios_stop_timer("create_qp", callTime); - - /* exchange data, converting info to network order and back */ - param_out.lid = htonl(transport_global_data.nic_lid); - param_out.req_qpn = htonl(c->req_qp.qpn); - - memset(&att, 0, sizeof(att)); - att.qp_context = c; - att.send_cq = transport_global_data.data_cq; - att.recv_cq = transport_global_data.data_cq; - att.srq = transport_global_data.data_srq; - att.cap.max_recv_wr = transport_global_data.qp_count; - att.cap.max_send_wr = transport_global_data.qp_count; - att.cap.max_recv_sge = 32; - att.cap.max_send_sge = 32; - att.qp_type = IBV_QPT_RC; - c->data_qp.qp = ibv_create_qp_wrapper(transport_global_data.pd, &att); - if (!c->data_qp.qp) { - log_error(nnti_debug_level, "failed to create QP: %s", strerror(errno)); - } - if (ibv_req_notify_cq_wrapper(transport_global_data.data_cq, 0)) { - log_error(nnti_debug_level, "Couldn't request CQ notification: %s", strerror(errno)); - } - - c->data_qp.qpn = c->data_qp.qp->qp_num; - param_out.my_qpn = htonl(c->data_qp.qpn); - - param_out.atomics_rkey=transport_global_data.atomics_mr->rkey; - param_out.atomics_addr=(uint64_t)transport_global_data.atomics_mr->addr; - - trios_start_timer(callTime); - rc = tcp_exchange(sock, 0, ¶m_in, ¶m_out, sizeof(param_in)); - trios_stop_timer("exch data", callTime); - if (rc) - goto out; - - c->peer_name = strdup(param_in.name); - c->peer_addr = ntohl(param_in.addr); - c->peer_port = ntohl(param_in.port); - c->peer_lid = ntohl(param_in.lid); - c->peer_req_qpn = ntohl(param_in.req_qpn); - c->data_qp.peer_qpn = ntohl(param_in.my_qpn); - c->atomics_rkey = param_in.atomics_rkey; - c->atomics_addr = param_in.atomics_addr; - -out: - return rc; -} - -static int new_server_connection( - ib_connection *c, - int sock) -{ - int rc; -// int flags=0; - struct ibv_qp_init_attr att; - - /* - * IB parameters passed through TCP to establish the IB connection. - */ - struct { - char name[NNTI_HOSTNAME_LEN]; - uint32_t addr; - uint32_t port; - uint32_t lid; - uint32_t req_qpn; - uint32_t my_qpn; - uint32_t peer_qpn; - uint32_t atomics_rkey; - uint64_t atomics_addr; - } param_in, param_out; - - // initialize structs to avoid valgrind warnings - memset(¶m_out, 0, sizeof(param_out)); - memset(¶m_in, 0, sizeof(param_in)); - - trios_declare_timer(callTime); - - strcpy(param_out.name, transport_global_data.listen_name); - param_out.addr = htonl((uint32_t)transport_global_data.listen_addr); - param_out.port = htonl((uint32_t)transport_global_data.listen_port); - - /* create the main queue pair */ - memset(&att, 0, sizeof(att)); - att.qp_context = c; - att.send_cq = transport_global_data.req_cq; - att.recv_cq = transport_global_data.req_cq; - att.srq = transport_global_data.req_srq; - att.cap.max_recv_wr = transport_global_data.qp_count; - att.cap.max_send_wr = transport_global_data.qp_count; - att.cap.max_recv_sge = 32; - att.cap.max_send_sge = 32; - att.qp_type = IBV_QPT_RC; - - trios_start_timer(callTime); - c->req_qp.qp = ibv_create_qp_wrapper(transport_global_data.pd, &att); - if (!c->req_qp.qp) { - log_error(nnti_debug_level, "failed to create QP: %s", strerror(errno)); - } - c->req_qp.qpn = c->req_qp.qp->qp_num; - trios_stop_timer("create_qp", callTime); - - /* exchange data, converting info to network order and back */ - param_out.lid = htonl(transport_global_data.nic_lid); - param_out.req_qpn = htonl(c->req_qp.qpn); - - memset(&att, 0, sizeof(att)); - att.qp_context = c; - att.send_cq = transport_global_data.data_cq; - att.recv_cq = transport_global_data.data_cq; - att.srq = transport_global_data.data_srq; - att.cap.max_recv_wr = transport_global_data.qp_count; - att.cap.max_send_wr = transport_global_data.qp_count; - att.cap.max_recv_sge = 32; - att.cap.max_send_sge = 32; - att.qp_type = IBV_QPT_RC; - c->data_qp.qp = ibv_create_qp_wrapper(transport_global_data.pd, &att); - if (!c->data_qp.qp) { - log_error(nnti_debug_level, "failed to create QP: %s", strerror(errno)); - } - if (ibv_req_notify_cq_wrapper(transport_global_data.data_cq, 0)) { - log_error(nnti_debug_level, "Couldn't request CQ notification: %s", strerror(errno)); - } - - c->data_qp.qpn = c->data_qp.qp->qp_num; - param_out.my_qpn = htonl(c->data_qp.qpn); - - param_out.atomics_rkey=transport_global_data.atomics_mr->rkey; - param_out.atomics_addr=(uint64_t)transport_global_data.atomics_mr->addr; - - trios_start_timer(callTime); - rc = tcp_exchange(sock, 1, ¶m_in, ¶m_out, sizeof(param_in)); - trios_stop_timer("exch data", callTime); - if (rc) - goto out; - - c->peer_name = strdup(param_in.name); - c->peer_addr = ntohl(param_in.addr); - c->peer_port = ntohl(param_in.port); - c->peer_lid = ntohl(param_in.lid); - c->peer_req_qpn = ntohl(param_in.req_qpn); - c->data_qp.peer_qpn = ntohl(param_in.my_qpn); - c->atomics_rkey = param_in.atomics_rkey; - c->atomics_addr = param_in.atomics_addr; - -out: - return rc; -} - -static NNTI_result_t insert_conn_peer(const NNTI_peer_t *peer, ib_connection *conn) -{ - NNTI_result_t rc=NNTI_OK; - addrport_key key; - - key.addr = peer->peer.NNTI_remote_process_t_u.ib.addr; - key.port = peer->peer.NNTI_remote_process_t_u.ib.port; - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "insert_conn_peer", peer); - } - - nthread_lock(&nnti_conn_peer_lock); -// assert(connections_by_peer.find(key) == connections_by_peer.end()); - if (connections_by_peer.find(key) == connections_by_peer.end()) { - connections_by_peer[key] = conn; // add to connection map - } - nthread_unlock(&nnti_conn_peer_lock); - - log_debug(nnti_debug_level, "peer connection added (conn=%p)", conn); - -// print_peer_map(); - - return(rc); -} -static NNTI_result_t insert_conn_qpn(const NNTI_qp_num qpn, ib_connection *conn) -{ - NNTI_result_t rc=NNTI_OK; - - nthread_lock(&nnti_conn_qpn_lock); - assert(connections_by_qpn.find(qpn) == connections_by_qpn.end()); - connections_by_qpn[qpn] = conn; - nthread_unlock(&nnti_conn_qpn_lock); - - log_debug(nnti_debug_level, "qpn connection added (conn=%p)", conn); - -// print_qpn_map(); - - return(rc); -} -static ib_connection *get_conn_peer(const NNTI_peer_t *peer) -{ - ib_connection *conn = NULL; - - addrport_key key; - - assert(peer); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "get_conn_peer", peer); - } - - memset(&key, 0, sizeof(addrport_key)); - key.addr=peer->peer.NNTI_remote_process_t_u.ib.addr; - key.port=peer->peer.NNTI_remote_process_t_u.ib.port; - - nthread_lock(&nnti_conn_peer_lock); - if (connections_by_peer.find(key) != connections_by_peer.end()) { - conn = connections_by_peer[key]; - } - nthread_unlock(&nnti_conn_peer_lock); - -// print_peer_map(); - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - return conn; - } - - log_debug(nnti_debug_level, "connection NOT found"); -// print_peer_map(); - - return(NULL); -} -static ib_connection *get_conn_qpn(const NNTI_qp_num qpn) -{ - ib_connection *conn=NULL; - - log_debug(nnti_debug_level, "looking for qpn=%llu", (unsigned long long)qpn); - nthread_lock(&nnti_conn_qpn_lock); - if (connections_by_qpn.find(qpn) != connections_by_qpn.end()) { - conn = connections_by_qpn[qpn]; - } - nthread_unlock(&nnti_conn_qpn_lock); - -// print_qpn_map(); - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - return conn; - } - - log_debug(nnti_debug_level, "connection NOT found"); -// print_qpn_map(); - - return(NULL); -} -static NNTI_peer_t *get_peer_by_url(const char *url) -{ - ib_connection *conn = NULL; - - log_debug(nnti_debug_level, "looking for url=%s", url); - - conn_by_peer_iter_t i; - nthread_lock(&nnti_conn_peer_lock); - for (i=connections_by_peer.begin(); i != connections_by_peer.end(); i++) { - log_debug(nnti_debug_level, "peer_map key=(%llu,%llu) conn=%p", - (uint64_t)i->first.addr, (uint64_t)i->first.port, i->second); - if (strcmp(i->second->peer.url, url) == 0) { - conn=i->second; - break; - } - } - nthread_unlock(&nnti_conn_peer_lock); - - if (conn != NULL) { - log_debug(nnti_debug_level, "peer found"); - NNTI_peer_t *peer=(NNTI_peer_t*)malloc(sizeof(NNTI_peer_t)); - *peer=conn->peer; - return peer; - } - - return(NULL); -} -static ib_connection *del_conn_peer(const NNTI_peer_t *peer) -{ - ib_connection *conn=NULL; - addrport_key key; - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "del_conn_peer", peer); - } - - memset(&key, 0, sizeof(addrport_key)); - key.addr=peer->peer.NNTI_remote_process_t_u.ib.addr; - key.port=peer->peer.NNTI_remote_process_t_u.ib.port; - - nthread_lock(&nnti_conn_peer_lock); - if (connections_by_peer.find(key) != connections_by_peer.end()) { - conn = connections_by_peer[key]; - } - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - connections_by_peer.erase(key); - del_conn_qpn(conn->req_qp.qpn); - del_conn_qpn(conn->data_qp.qpn); - } else { - log_debug(nnti_debug_level, "connection NOT found"); - } - nthread_unlock(&nnti_conn_peer_lock); - - return(conn); -} -static ib_connection *del_conn_qpn(const NNTI_qp_num qpn) -{ - ib_connection *conn=NULL; - - nthread_lock(&nnti_conn_qpn_lock); - if (connections_by_qpn.find(qpn) != connections_by_qpn.end()) { - conn = connections_by_qpn[qpn]; - } - - if (conn != NULL) { - log_debug(nnti_debug_level, "connection found"); - connections_by_qpn.erase(qpn); - } else { - log_debug(nnti_debug_level, "connection NOT found"); - } - nthread_unlock(&nnti_conn_qpn_lock); - - return(conn); -} -//static void print_qpn_map() -//{ -// ib_connection *conn=NULL; -// conn_by_qpn_iter_t i; -// for (i=connections_by_qpn.begin(); i != connections_by_qpn.end(); i++) { -// conn=i->second; -// log_debug(nnti_debug_level, "qpn_map key=%llu conn=%p (name=%s, addr=%llu, port=%llu)", -// i->first, conn, conn->peer_name, (uint64_t)conn->peer_addr, (uint64_t)conn->peer_port); -// } -//} -//static void print_peer_map() -//{ -// ib_connection *conn=NULL; -// conn_by_peer_iter_t i; -// for (i=connections_by_peer.begin(); i != connections_by_peer.end(); i++) { -// addrport_key key=i->first; -// conn=i->second; -// log_debug(nnti_debug_level, "peer_map key=(%llu,%llu) conn=%p (name=%s, addr=%llu, port=%llu)", -// (uint64_t)key.addr, (uint64_t)key.port, conn, conn->peer_name, (uint64_t)conn->peer_addr, (uint64_t)conn->peer_port); -// } -//} - -static NNTI_result_t insert_buf_bufhash(NNTI_buffer_t *buf) -{ - NNTI_result_t rc=NNTI_OK; - uint32_t h=hash6432shift((uint64_t)buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - - nthread_lock(&nnti_buf_bufhash_lock); - assert(buffers_by_bufhash.find(h) == buffers_by_bufhash.end()); - buffers_by_bufhash[h] = buf; - nthread_unlock(&nnti_buf_bufhash_lock); - - log_debug(nnti_debug_level, "bufhash buffer added (buf=%p bufhash=%x)", buf, (uint64_t)h); - - return(rc); -} -static NNTI_buffer_t *get_buf_bufhash(const uint32_t bufhash) -{ - NNTI_buffer_t *buf=NULL; - - log_debug(nnti_debug_level, "looking for bufhash=%x", (uint64_t)bufhash); - nthread_lock(&nnti_buf_bufhash_lock); - if (buffers_by_bufhash.find(bufhash) != buffers_by_bufhash.end()) { - buf = buffers_by_bufhash[bufhash]; - } - nthread_unlock(&nnti_buf_bufhash_lock); - - if (buf != NULL) { - log_debug(nnti_debug_level, "buffer found (buf=%p)", buf); - return buf; - } - - log_debug(nnti_debug_level, "buffer NOT found"); -// print_bufhash_map(); - - return(NULL); -} -static NNTI_buffer_t *del_buf_bufhash(NNTI_buffer_t *buf) -{ - uint32_t h=hash6432shift((uint64_t)buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf); - log_level debug_level = nnti_debug_level; - - nthread_lock(&nnti_buf_bufhash_lock); - if (buffers_by_bufhash.find(h) != buffers_by_bufhash.end()) { - buf = buffers_by_bufhash[h]; - } - - if (buf != NULL) { - log_debug(debug_level, "buffer found"); - buffers_by_bufhash.erase(h); - } else { - log_debug(debug_level, "buffer NOT found"); - } - nthread_unlock(&nnti_buf_bufhash_lock); - - return(buf); -} -//static void print_bufhash_map() -//{ -// if (!logging_debug(nnti_debug_level)) { -// return; -// } -// -// if (buffers_by_bufhash.empty()) { -// log_debug(nnti_debug_level, "bufhash_map is empty"); -// return; -// } -// -// buf_by_bufhash_iter_t i; -// for (i=buffers_by_bufhash.begin(); i != buffers_by_bufhash.end(); i++) { -// log_debug(nnti_debug_level, "bufhash_map key=%x buf=%p", i->first, i->second); -// } -//} - -static NNTI_result_t wr_pool_register( - ib_work_request *ib_wr) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - - trios_declare_timer(callTime); - - struct ibv_mr *mr=NULL; - - uint32_t len; - - log_debug(nnti_debug_level, "enter"); - - len = sizeof(ib_wr->ack); - - if (config.use_memset) { - trios_start_timer(callTime); - memset(&ib_wr->ack, 0, len); - trios_stop_timer("memset", callTime); - } - if (config.use_mlock) { - trios_start_timer(callTime); - mlock(&ib_wr->ack, len); - trios_stop_timer("mlock", callTime); - } - - trios_start_timer(callTime); - mr = ibv_reg_mr_wrapper( - transport_global_data.pd, - &ib_wr->ack, - len, - (ibv_access_flags)(IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE)); - if (!mr) { - log_error(nnti_debug_level, "failed to register memory region"); - perror("errno"); - return NNTI_EIO; - } - trios_stop_timer("register", callTime); - - ib_wr->ack_mr=mr; - - log_debug(nnti_debug_level, "exit (mr==%p, addr %p, length %lu, lkey %x, rkey %x)...", mr, mr->addr, mr->length, mr->lkey, mr->rkey); - - return (rc); -} -static NNTI_result_t wr_pool_deregister( - ib_work_request *ib_wr) -{ - int ibv_rc=0; - trios_declare_timer(callTime); - - log_debug(nnti_debug_level, "enter"); - - if (ib_wr->ack_mr!=NULL) { - trios_start_timer(callTime); - ibv_rc=ibv_dereg_mr_wrapper(ib_wr->ack_mr); - if (ibv_rc != 0) { - log_error(nnti_debug_level, "deregistering the ACK buffer failed"); - } - trios_stop_timer("deregister", callTime); - - if (config.use_mlock) { - trios_start_timer(callTime); - munlock(ib_wr->ack_mr->addr, ib_wr->ack_mr->length); - trios_stop_timer("munlock", callTime); - } - - ib_wr->ack_mr=NULL; - } else { - log_debug(nnti_debug_level, "exit ib_wr(%p) - not registered", ib_wr); - return(NNTI_OK); - } - - log_debug(nnti_debug_level, "exit"); - - return(NNTI_OK); -} -static NNTI_result_t wr_pool_init(uint32_t pool_size) -{ - NNTI_result_t rc=NNTI_OK; - uint32_t i; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - for (i=0;ilock); - rc=wr_pool_register(ib_wr); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "failed to register target work request: rc=%d", rc); - goto cleanup; - } - wr_pool_rdma_push(ib_wr); - - ib_wr=(ib_work_request *)calloc(1, sizeof(ib_work_request)); - assert(ib_wr); - log_debug(nnti_debug_level, "allocated ib_wr (ib_wr=%p)", ib_wr); - nthread_lock_init(&ib_wr->lock); - wr_pool_sendrecv_push(ib_wr); - } - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(rc); -} -static ib_work_request *wr_pool_rdma_pop(void) -{ - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - nthread_lock(&nnti_wr_pool_lock); - if (!rdma_wr_pool.empty()) { - ib_wr=rdma_wr_pool.front(); - rdma_wr_pool.pop_front(); - } - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return(ib_wr); -} -static ib_work_request *wr_pool_sendrecv_pop(void) -{ - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - nthread_lock(&nnti_wr_pool_lock); - if (!sendrecv_wr_pool.empty()) { - ib_wr=sendrecv_wr_pool.front(); - sendrecv_wr_pool.pop_front(); - } - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return(ib_wr); -} -static void wr_pool_rdma_push(ib_work_request *ib_wr) -{ - log_debug(nnti_debug_level, "enter"); - - ib_wr->last_op=0; - ib_wr->state =NNTI_IB_WR_STATE_RESET; - - nthread_lock(&nnti_wr_pool_lock); - rdma_wr_pool.push_front(ib_wr); - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return; -} -static void wr_pool_sendrecv_push(ib_work_request *ib_wr) -{ - log_debug(nnti_debug_level, "enter"); - - ib_wr->last_op=0; - ib_wr->state =NNTI_IB_WR_STATE_RESET; - - nthread_lock(&nnti_wr_pool_lock); - sendrecv_wr_pool.push_front(ib_wr); - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return; -} -static NNTI_result_t wr_pool_fini(void) -{ - NNTI_result_t rc=NNTI_OK; - ib_work_request *ib_wr=NULL; - - log_debug(nnti_debug_level, "enter"); - - nthread_lock(&nnti_wr_pool_lock); - while (!rdma_wr_pool.empty()) { - ib_wr=rdma_wr_pool.front(); - rdma_wr_pool.pop_front(); - assert(ib_wr); - rc=wr_pool_deregister(ib_wr); - if (rc!=NNTI_OK) { - log_error(nnti_debug_level, "failed to deregister target work request: rc=%d", rc); - goto cleanup; - } - log_debug(nnti_debug_level, "freeing ib_wr=%p", ib_wr); - free(ib_wr); - } - while (!sendrecv_wr_pool.empty()) { - ib_wr=sendrecv_wr_pool.front(); - sendrecv_wr_pool.pop_front(); - assert(ib_wr); - log_debug(nnti_debug_level, "freeing ib_wr=%p", ib_wr); - free(ib_wr); - } - -cleanup: - nthread_unlock(&nnti_wr_pool_lock); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - -static void close_all_conn(void) -{ - log_level debug_level = nnti_debug_level; - - log_debug(nnti_debug_level, "enter (%d qpn connections, %d peer connections)", - connections_by_qpn.size(), connections_by_peer.size()); - - nthread_lock(&nnti_conn_qpn_lock); - conn_by_qpn_iter_t qpn_iter = connections_by_qpn.begin(); - while (qpn_iter != connections_by_qpn.end()) { - log_debug(debug_level, "close connection (qpn=%llu)", qpn_iter->first); - close_connection(qpn_iter->second); - - connections_by_qpn.erase(qpn_iter++); - } - - nthread_unlock(&nnti_conn_qpn_lock); - - nthread_lock(&nnti_conn_peer_lock); - conn_by_peer_iter_t peer_iter = connections_by_peer.begin(); - while (peer_iter != connections_by_peer.end()) { - log_debug(debug_level, "close connection (peer.addr=%llu)", peer_iter->first.addr); -// close_connection(peer_iter->second); - - connections_by_peer.erase(peer_iter++); - } - nthread_unlock(&nnti_conn_peer_lock); - - log_debug(debug_level, "exit (%d qpn connections, %d peer connections)", - connections_by_qpn.size(), connections_by_peer.size()); - - return; -} - -/** - * @brief initialize - */ -static NNTI_result_t init_connection( - ib_connection **conn, - const int sock, - const int is_server) -{ - int rc=0; /* return code */ - - trios_declare_timer(callTime); - - log_debug(nnti_debug_level, "initializing ib connection"); - - (*conn)->disconnect_requested = FALSE; - - trios_start_timer(callTime); - if (is_server) { - rc = new_server_connection(*conn, sock); - } else { - rc = new_client_connection(*conn, sock); - } - if (rc) { - close_connection(*conn); - goto out; - } - trios_stop_timer("new connection", callTime); - - print_ib_conn(*conn); - -out: - return((NNTI_result_t)rc); -} - -/* - * At an explicit BYE message, or at finalize time, shut down a connection. - * If descriptors are posted, defer and clean up the connection structures - * later. - */ -static void close_connection(ib_connection *c) -{ - int rc; -// int i; - - if (c==NULL) return; - - log_debug(nnti_debug_level, "enter"); - - if (c->state==DISCONNECTED) { - log_debug(nnti_debug_level, "conn(%p) is already closed", c); - return; - } - - print_ib_conn(c); - - transition_connection_to_error(c); - - if (c->peer_name) free(c->peer_name); - if (c->req_qp.qp) { - rc=ibv_destroy_qp_wrapper(c->req_qp.qp); - if (rc < 0) - log_error(nnti_debug_level, "failed to destroy QP"); - } - if (c->data_qp.qp) { - rc=ibv_destroy_qp_wrapper(c->data_qp.qp); - if (rc < 0) - log_error(nnti_debug_level, "failed to destroy QP"); - } - c->state=DISCONNECTED; - - log_debug(nnti_debug_level, "exit"); -} - -static NNTI_result_t check_for_waiting_connection() -{ - NNTI_result_t rc = NNTI_OK; - - struct sockaddr_in ssin; - socklen_t len; - int s; - NNTI_peer_t peer; // this doesn't need to be a pointer - ib_connection *conn = NULL; - - len = sizeof(ssin); - s = accept(transport_global_data.listen_sock, (struct sockaddr *) &ssin, &len); - if (s < 0) { - if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) { - log_debug(nnti_debug_level, "no connections waiting to be accepted: %s", strerror(errno)); - rc = NNTI_EWOULDBLOCK; - goto cleanup; - } else { - log_error(nnti_debug_level, "failed to accept tcp socket connection: %s", strerror(errno)); - rc = NNTI_EIO; - goto cleanup; - } - } else { - static int connection_count = 0; - char *peer_hostname = strdup(inet_ntoa(ssin.sin_addr)); -// NNTI_ip_addr peer_addr = ssin.sin_addr.s_addr; - NNTI_tcp_port peer_port = ntohs(ssin.sin_port); - - conn = (ib_connection *)calloc(1, sizeof(ib_connection)); // TODO: FIND OUT WHERE THIS IS FREED - log_debug(nnti_debug_level, "calloc returned conn=%p.", conn); - if (conn == NULL) { - log_error(nnti_debug_level, "calloc returned NULL. out of memory?: %s", strerror(errno)); - rc=NNTI_ENOMEM; - goto cleanup; - } - -// nthread_lock(&nnti_ib_lock); - rc=init_connection(&conn, s, 1); - if (rc!=NNTI_OK) { - goto cleanup; - } - create_peer( - &peer, - conn->peer_name, - conn->peer_addr, - conn->peer_port); - - conn->peer=peer; - - insert_conn_qpn(conn->req_qp.qpn, conn); - insert_conn_qpn(conn->data_qp.qpn, conn); - insert_conn_peer(&peer, conn); - - log_debug(nnti_debug_level, "Allocating new connection count=%d", ++connection_count); - - transition_connection_to_ready(s, conn); -// nthread_unlock(&nnti_ib_lock); - - log_debug(nnti_debug_level, "accepted new connection from %s:%u", peer_hostname, peer_port); - if (peer_hostname) free(peer_hostname); // fixed to avoid lost bytes reported by valgrind - - if (close(s) < 0) { - log_error(nnti_debug_level, "failed to close new tcp socket"); - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "peer", - "end of check_listen_socket_for_new_connections", &peer); - } - } - -cleanup: - return rc; -} - -/** - * Check for new connections. The listening socket is left nonblocking - * so this test can be quick; but accept is not really that quick compared - * to polling an IB interface, for instance. Returns >0 if an accept worked. - */ -static NNTI_result_t check_listen_socket_for_new_connections() -{ - bool done=false; - trios_declare_timer(callTime); - - trios_start_timer(callTime); - while(!done) { - if (check_for_waiting_connection() != NNTI_OK) { - done=true; - } - } - trios_stop_timer("check_for_waiting_connection", callTime); - - return(NNTI_OK); -} - - -static struct ibv_device *get_ib_device(void) -{ - struct ibv_device *dev; - struct ibv_device **dev_list; - int dev_count=0; - - dev_list = ibv_get_device_list_wrapper(&dev_count); - if (dev_count == 0) - return NULL; - if (dev_count > 1) { - log_debug(nnti_debug_level, "found %d devices, defaulting the dev_list[0] (%p)", dev_count, dev_list[0]); - } - dev = dev_list[0]; - ibv_free_device_list_wrapper(dev_list); - - return dev; -} - -static void get_qp_state(struct ibv_qp *qp) -{ - struct ibv_qp_attr attr; - struct ibv_qp_init_attr init_attr; - - if (ibv_query_qp(qp, &attr, - IBV_QP_STATE, &init_attr)) { - log_error(nnti_debug_level, "Failed to query QP state\n"); - return; - } - log_debug(nnti_debug_level, "qp(%p)->state=%d", qp, attr.qp_state); -} - -static void print_all_qp_state(void) -{ - if (logging_debug(nnti_debug_level)) { - ib_connection *conn=NULL; - conn_by_qpn_iter_t i=connections_by_qpn.begin(); - if (i != connections_by_qpn.end()) { - conn=i->second; - get_qp_state(conn->req_qp.qp); - } - } -} - -static void print_wc(const struct ibv_wc *wc, bool force) -{ - if (force) { - log_debug(LOG_ALL, "wc=%p, wc.opcode=%d, wc.flags=%d, wc.status=%d (%s), wc.wr_id=%lx, wc.vendor_err=%u, wc.byte_len=%u, wc.qp_num=%u, wc.imm_data=%x, wc.src_qp=%u", - wc, - wc->opcode, - wc->wc_flags, - wc->status, - ibv_wc_status_str_wrapper(wc->status), - wc->wr_id, - wc->vendor_err, - wc->byte_len, - wc->qp_num, - wc->imm_data, - wc->src_qp); - } else if ((wc->status != IBV_WC_SUCCESS) && (wc->status != IBV_WC_RNR_RETRY_EXC_ERR)) { - log_error(nnti_debug_level, "wc=%p, wc.opcode=%d, wc.flags=%d, wc.status=%d (%s), wc.wr_id=%lx, wc.vendor_err=%u, wc.byte_len=%u, wc.qp_num=%u, wc.imm_data=%x, wc.src_qp=%u", - wc, - wc->opcode, - wc->wc_flags, - wc->status, - ibv_wc_status_str_wrapper(wc->status), - wc->wr_id, - wc->vendor_err, - wc->byte_len, - wc->qp_num, - wc->imm_data, - wc->src_qp); - } else { - log_debug(nnti_debug_level, "wc=%p, wc.opcode=%d, wc.flags=%d, wc.status=%d (%s), wc.wr_id=%lx, wc.vendor_err=%u, wc.byte_len=%u, wc.qp_num=%u, wc.imm_data=%x, wc.src_qp=%u", - wc, - wc->opcode, - wc->wc_flags, - wc->status, - ibv_wc_status_str_wrapper(wc->status), - wc->wr_id, - wc->vendor_err, - wc->byte_len, - wc->qp_num, - wc->imm_data, - wc->src_qp); - } -} - -static NNTI_result_t process_comp_channel_event( - struct ibv_comp_channel *comp_channel, - struct ibv_cq *cq) -{ - NNTI_result_t rc=NNTI_OK; - - int retries_left=3; - - struct ibv_cq *ev_cq; - void *ev_ctx; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - - log_debug(nnti_debug_level, "enter"); - - trios_start_timer(call_time); - trios_start_timer(total_time); - -try_again: - if (ibv_get_cq_event_wrapper(comp_channel, &ev_cq, &ev_ctx) == 0) { - trios_stop_timer("ibv_get_cq_event", call_time); - log_debug(nnti_debug_level, "got event from comp_channel=%p for cq=%p", comp_channel, ev_cq); - trios_start_timer(call_time); - ibv_ack_cq_events_wrapper(ev_cq, 1); - trios_stop_timer("ibv_ack_cq_events", call_time); - log_debug(nnti_debug_level, "ACKed event on cq=%p", ev_cq); - rc = NNTI_OK; - } else { - if (errno == EAGAIN) { - if (retries_left > 0) { - trios_stop_timer("ibv_get_cq_event...EAGAIN...retry", call_time); - retries_left--; - trios_start_timer(call_time); - goto try_again; - } else { - trios_stop_timer("ibv_get_cq_event...EAGAIN...retries exhausted", call_time); - rc = NNTI_EAGAIN; - goto cleanup; - } - } - trios_stop_timer("ibv_get_cq_event...failed", call_time); - log_error(nnti_debug_level, "ibv_get_cq_event failed (ev_cq==%p): %s", - ev_cq, strerror(errno)); - rc = NNTI_EIO; - goto cleanup; - } - -cleanup: - trios_stop_timer("process_comp_channel_event", total_time); - - log_debug(nnti_debug_level, "exit"); - - return(rc); -} - -#define CONNECTION_SOCKET_INDEX 0 -#define DATA_CQ_SOCKET_INDEX 1 -#define REQ_CQ_SOCKET_INDEX 2 -#define INTERRUPT_PIPE_INDEX 3 -#define FD_COUNT 4 -static NNTI_result_t poll_all( - int timeout) -{ - NNTI_result_t rc=NNTI_OK; - int poll_rc=0; - struct pollfd my_pollfd[FD_COUNT]; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - - log_debug(nnti_debug_level, "enter (timeout=%d)", timeout); - - trios_start_timer(total_time); - - /* - * poll the channel until it has an event and sleep ms_timeout - * milliseconds between any iteration - */ - my_pollfd[CONNECTION_SOCKET_INDEX].fd = transport_global_data.listen_sock; - my_pollfd[CONNECTION_SOCKET_INDEX].events = POLLIN; - my_pollfd[CONNECTION_SOCKET_INDEX].revents = 0; - my_pollfd[DATA_CQ_SOCKET_INDEX].fd = transport_global_data.data_comp_channel->fd; - my_pollfd[DATA_CQ_SOCKET_INDEX].events = POLLIN; - my_pollfd[DATA_CQ_SOCKET_INDEX].revents = 0; - my_pollfd[REQ_CQ_SOCKET_INDEX].fd = transport_global_data.req_comp_channel->fd; - my_pollfd[REQ_CQ_SOCKET_INDEX].events = POLLIN; - my_pollfd[REQ_CQ_SOCKET_INDEX].revents = 0; - my_pollfd[INTERRUPT_PIPE_INDEX].fd = transport_global_data.interrupt_pipe[0]; - my_pollfd[INTERRUPT_PIPE_INDEX].events = POLLIN; - my_pollfd[INTERRUPT_PIPE_INDEX].revents = 0; - log_debug(nnti_debug_level, "polling with timeout==%d", timeout); - - // Test for errno==EINTR to deal with timing interrupts from HPCToolkit - do { - trios_start_timer(call_time); - poll_rc = poll(&my_pollfd[0], FD_COUNT, timeout); - trios_stop_timer("poll", call_time); - } while ((poll_rc < 0) && (errno == EINTR)); - - if (poll_rc == 0) { - log_debug(nnti_debug_level, "poll() timed out: poll_rc=%d", poll_rc); - rc = NNTI_ETIMEDOUT; - goto cleanup; - } else if (poll_rc < 0) { - if (errno == EINTR) { - log_error(nnti_debug_level, "poll() interrupted by signal: poll_rc=%d (%s)", poll_rc, strerror(errno)); - rc = NNTI_EINTR; - } else if (errno == ENOMEM) { - log_error(nnti_debug_level, "poll() out of memory: poll_rc=%d (%s)", poll_rc, strerror(errno)); - rc = NNTI_ENOMEM; - } else { - log_error(nnti_debug_level, "poll() invalid args: poll_rc=%d (%s)", poll_rc, strerror(errno)); - rc = NNTI_EINVAL; - } - goto cleanup; - } else { - log_debug(nnti_debug_level, "polled on %d file descriptor(s). events occurred on %d file descriptor(s).", FD_COUNT, poll_rc); - log_debug(nnti_debug_level, "poll success: poll_rc=%d ; my_pollfd[CONNECTION_SOCKET_INDEX].revents=%d", - poll_rc, my_pollfd[CONNECTION_SOCKET_INDEX].revents); - log_debug(nnti_debug_level, "poll success: poll_rc=%d ; my_pollfd[DATA_CQ_SOCKET_INDEX].revents=%d", - poll_rc, my_pollfd[DATA_CQ_SOCKET_INDEX].revents); - log_debug(nnti_debug_level, "poll success: poll_rc=%d ; my_pollfd[REQ_CQ_SOCKET_INDEX].revents=%d", - poll_rc, my_pollfd[REQ_CQ_SOCKET_INDEX].revents); - log_debug(nnti_debug_level, "poll success: poll_rc=%d ; my_pollfd[INTERRUPT_PIPE_INDEX].revents=%d", - poll_rc, my_pollfd[INTERRUPT_PIPE_INDEX].revents); - } - - if (my_pollfd[CONNECTION_SOCKET_INDEX].revents == POLLIN) { - check_listen_socket_for_new_connections(); - } - if (my_pollfd[DATA_CQ_SOCKET_INDEX].revents == POLLIN) { - process_comp_channel_event(transport_global_data.data_comp_channel, transport_global_data.data_cq); - } - if (my_pollfd[REQ_CQ_SOCKET_INDEX].revents == POLLIN) { - process_comp_channel_event(transport_global_data.req_comp_channel, transport_global_data.req_cq); - } - if (my_pollfd[INTERRUPT_PIPE_INDEX].revents == POLLIN) { - log_debug(nnti_debug_level, "poll() interrupted by NNTI_ib_interrupt"); - // read all bytes from the pipe - ssize_t bytes_read=0; - uint32_t dummy=0; - do { - bytes_read=read(transport_global_data.interrupt_pipe[0], &dummy, 4); - if (dummy != 0xAAAAAAAA) { - log_warn(nnti_debug_level, "interrupt byte is %X, should be 0xAAAAAAAA", dummy); - } - log_debug(nnti_debug_level, "bytes_read==%lu", (uint64_t)bytes_read); - } while(bytes_read > 0); - rc = NNTI_EINTR; - } - if (ibv_req_notify_cq_wrapper(transport_global_data.data_cq, 0)) { - log_error(nnti_debug_level, "Couldn't request CQ notification: %s", strerror(errno)); - rc = NNTI_EIO; - } - if (ibv_req_notify_cq_wrapper(transport_global_data.req_cq, 0)) { - log_error(nnti_debug_level, "Couldn't request CQ notification: %s", strerror(errno)); - rc = NNTI_EIO; - } - -cleanup: - trios_stop_timer("poll_all", total_time); - - log_debug(nnti_debug_level, "exit"); - return(rc); -} - - -#define CQ_COUNT 2 -static NNTI_result_t progress( - int timeout, - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - int ibv_rc=0; - - uint32_t which=0; - - struct ibv_wc wc; - struct ibv_comp_channel *comp_channel; - struct ibv_cq *cq_list[CQ_COUNT]; - static int cq_index=0; - - long entry_time =trios_get_time_ms(); - long elapsed_time=0; - - log_level debug_level =nnti_debug_level; - log_level old_log_level=logger_get_default_level(); - - static bool in_progress =false; // if true, another thread is already making progress. - bool made_progress=false; - int8_t wr_complete =FALSE; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - - log_debug(debug_level, "enter (timeout=%d)", timeout); - - trios_start_timer(total_time); - - cq_list[0]=transport_global_data.data_cq; - cq_list[1]=transport_global_data.req_cq; - - /* - * Only one thread is allowed to make progress at a time. All others - * wait for the progress maker to finish, then everyone returns at once. - */ - nthread_lock(&nnti_progress_lock); - - wr_complete = is_any_wr_complete(wr_list, wr_count, &which); - - if (!in_progress) { - log_debug(debug_level, "making progress"); - // no other thread is making progress. we'll do it. - in_progress=true; - log_debug(debug_level, "set in_progress=true"); - nthread_unlock(&nnti_progress_lock); - } else { - if (wr_complete == TRUE) { - nthread_unlock(&nnti_progress_lock); - goto cleanup; - } - - // another thread is making progress. we'll wait until they are done. - rc=0; - elapsed_time=0; - if (in_progress) { - if (timeout > 0) { - log_debug(debug_level, "waiting for progress with timeout=%d", timeout-elapsed_time); - // wait for progress or until timeout - rc=nthread_timedwait(&nnti_progress_cond, &nnti_progress_lock, timeout-elapsed_time); - } else if (timeout < 0) { - log_debug(debug_level, "waiting infinitely for progress"); - // infinite wait for progress - rc=nthread_wait(&nnti_progress_cond, &nnti_progress_lock); - } else { - log_debug(debug_level, "waiting for progress with timeout=0. immediate timeout."); - // timeout == 0 and we are not the progress maker. report a timeout. - rc=ETIMEDOUT; - } - elapsed_time = (trios_get_time_ms() - entry_time); - log_debug(debug_level, "rc=%d, elapsed_time=%d", rc, elapsed_time); - } - nthread_unlock(&nnti_progress_lock); - if (rc == ETIMEDOUT) { - log_debug(debug_level, "timed out waiting for progress"); - nnti_rc = NNTI_ETIMEDOUT; - } else if (rc == 0) { - log_debug(debug_level, "someone made progress"); - nnti_rc=NNTI_OK; - } - goto cleanup; - } - - while (!made_progress) { - - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - nnti_rc=NNTI_ECANCELED; - break; - } - - check_listen_socket_for_new_connections(); - - for (int i=0;i 0) { - // got a work completion - log_debug(debug_level, "got wc from cq=%p", cq); - log_debug(debug_level, "polling status is %s", ibv_wc_status_str(wc.status)); - - print_wc(&wc, false); - if ((wc.status == IBV_WC_RNR_RETRY_EXC_ERR) || - (wc.status == IBV_WC_RETRY_EXC_ERR)) { - nnti_rc=NNTI_EDROPPED; - - ib_work_request *ib_wr=decode_work_request(&wc); - int min_rnr_timer=1; /* means 0.01ms delay before sending RNR NAK */ - int ack_timeout =17; /* time to wait for ACK/NAK before retransmitting. 4.096us * 2^17 == 0.536ss */ - int retry_count; - if (config.drop_if_full_queue) { - retry_count=1; /* number of retries if no answer on primary path or if remote sends RNR NAK */ - } else { - retry_count=7; /* number of retries if no answer on primary path or if remote sends RNR NAK. 7 has special meaning of infinite retries. */ - } - transition_qp_from_error_to_ready( - ib_wr->conn->req_qp.qp, - ib_wr->conn->peer_req_qpn, - ib_wr->conn->peer_lid, - min_rnr_timer, - ack_timeout, - retry_count); - - } else if (wc.status != IBV_WC_SUCCESS) { - log_error(debug_level, "Failed status %s (%d) for wr_id %lx", - ibv_wc_status_str(wc.status), - wc.status, wc.wr_id); - nnti_rc=NNTI_EIO; - } - - trios_start_timer(call_time); - ib_work_request *ib_wr=decode_work_request(&wc); - trios_stop_timer("progress - decode_work_request", call_time); - trios_start_timer(call_time); - nthread_lock(&ib_wr->lock); - process_event(ib_wr, &wc); - nthread_unlock(&ib_wr->lock); - trios_stop_timer("progress - process_event", call_time); - - made_progress=true; - } - } - - if ((!made_progress) && (wr_complete == FALSE)) { - trios_start_timer(call_time); - rc = poll_all(/*100*/ timeout-elapsed_time); - trios_stop_timer("progress - poll_all", call_time); - - elapsed_time = (trios_get_time_ms() - entry_time); - - /* case 1: success */ - if (rc == NNTI_OK) { - logger_set_default_level(old_log_level); - nnti_rc = NNTI_OK; - continue; - } - /* case 2: timed out */ - else if (rc==NNTI_ETIMEDOUT) { - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout >= 0) && (elapsed_time >= timeout)) || trios_exit_now()) { - logger_set_default_level(old_log_level); - log_debug(debug_level, "poll_all timed out"); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - /* continue if the timeout has not expired */ - log_debug(debug_level, "poll_all timedout... retrying"); - -// log_debug(debug_level, "***** disable debug logging. will enable after polling success. *****"); -// logger_set_default_level(LOG_OFF); - } - /* case 3: poll was interrupted. could be a signal or NNTI_interrupt(). */ - else if (rc==NNTI_EINTR) { - logger_set_default_level(old_log_level); - nnti_rc = NNTI_EINTR; - break; - } - /* case 4: poll out of memory */ - else if (rc==NNTI_ENOMEM) { - logger_set_default_level(old_log_level); - nnti_rc = NNTI_ENOMEM; - break; - } - /* case 5: poll got invalid arguments */ - else if (rc==NNTI_EINVAL) { - logger_set_default_level(old_log_level); - nnti_rc = NNTI_EINVAL; - break; - } - /* case 6: failure */ - else { - logger_set_default_level(old_log_level); - log_error(debug_level, "poll_all failed: %s", strerror(errno)); - nnti_rc = NNTI_EIO; - break; - } - } - } - -unlock: - nthread_lock(&nnti_progress_lock); - in_progress=false; - log_debug(debug_level, "set in_progress=false"); - nthread_broadcast(&nnti_progress_cond); - log_debug(debug_level, "broadcasted on nnti_progress_cond"); - nthread_unlock(&nnti_progress_lock); - -cleanup: - trios_stop_timer("progress", total_time); - - log_debug(debug_level, "exit"); - - return(nnti_rc); -} - - -static void print_ib_conn(ib_connection *c) -{ -// int i=0; - log_level debug_level=nnti_debug_level; - - log_debug(debug_level, "c->peer_name =%s", c->peer_name); - log_debug(debug_level, "c->peer_addr =%u", c->peer_addr); - log_debug(debug_level, "c->peer_port =%u", (uint32_t)c->peer_port); - log_debug(debug_level, "c->peer_lid =%llu", (uint64_t)c->peer_lid); - log_debug(debug_level, "c->peer_req_qpn =%llu", (uint64_t)c->peer_req_qpn); - - log_debug(debug_level, "c->req_comp_channel=%p", transport_global_data.req_comp_channel); - log_debug(debug_level, "c->req_cq =%p", transport_global_data.req_cq); - log_debug(debug_level, "c->req_srq =%p", transport_global_data.req_srq); - - log_debug(debug_level, "c->req_qp.qp =%p", c->req_qp.qp); - log_debug(debug_level, "c->req_qp.qpn =%llu", (uint64_t)c->req_qp.qpn); - - log_debug(debug_level, "c->data_qp.qp =%p", c->data_qp.qp); - log_debug(debug_level, "c->data_qp.qpn =%llu", (uint64_t)c->data_qp.qpn); - log_debug(debug_level, "c->data_qp.peer_qpn =%llu", (uint64_t)c->data_qp.peer_qpn); - - log_debug(debug_level, "c->state =%d", c->state); - - log_debug(debug_level, "c->disconnect_requested=%d", c->disconnect_requested); - -} - -static void config_init(nnti_ib_config *c) -{ - c->min_atomics_vars = 512; - c->use_wr_pool = false; - c->use_rdma_target_ack = false; - c->use_mlock = true; - c->use_memset = true; - c->drop_if_full_queue = false; -} - -static void config_get_from_env(nnti_ib_config *c) -{ - char *env_str=NULL; - - if ((env_str=getenv("TRIOS_NNTI_MIN_ATOMIC_VARS")) != NULL) { - errno=0; - uint32_t min_vars=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->min_atomics_vars to %lu", min_vars); - c->min_atomics_vars=min_vars; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MIN_ATOMIC_VARS value conversion failed (%s). using c->min_atomics_vars default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MIN_ATOMIC_VARS is undefined. using c->min_atomics_vars default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_WR_POOL")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_wr_pool to TRUE"); - c->use_wr_pool=true; - } else { - log_debug(nnti_debug_level, "setting c->use_wr_pool to FALSE"); - c->use_wr_pool=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_WR_POOL is undefined. using c->use_wr_pool default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_RDMA_TARGET_ACK")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(nnti_debug_level, "setting c->use_rdma_target_ack to TRUE"); - c->use_rdma_target_ack=true; - } else { - log_debug(nnti_debug_level, "setting c->use_rdma_target_ack to FALSE"); - c->use_rdma_target_ack=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_RDMA_TARGET_ACK is undefined. using c->use_rdma_target_ack default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_MLOCK")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(LOG_ALL, "setting c->use_mlock to TRUE"); - c->use_mlock=true; - } else { - log_debug(LOG_ALL, "setting c->use_mlock to FALSE"); - c->use_mlock=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_MLOCK is undefined. using c->mlock default"); - } - if ((env_str=getenv("TRIOS_NNTI_USE_MEMSET")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(LOG_ALL, "setting c->use_memset to TRUE"); - c->use_memset=true; - } else { - log_debug(LOG_ALL, "setting c->use_memset to FALSE"); - c->use_memset=false; - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_USE_MEMSET is undefined. using c->memset default"); - } -} - -//static void print_wr(ib_work_request *ib_wr) -//{ -// log_debug(nnti_debug_level, "ib_wr (op=%llu ; offset=%llu ; length=%llu)", -// (uint64_t)ib_wr->last_op, -// ib_wr->offset, -// ib_wr->length); -//} -// -//static void print_ack_buf(ib_rdma_ack *ack) -//{ -// log_debug(nnti_debug_level, "ack (op=%llu ; offset=%llu ; length=%llu)", -// (uint64_t)ack->op, -// ack->offset, -// ack->length); -//} -// -//static void print_xfer_buf(void *buf, uint32_t size) -//{ -// struct data_t { -// uint32_t int_val; -// float float_val; -// double double_val; -// }; -// -// struct data_array_t { -// u_int data_array_t_len; -// struct data_t *data_array_t_val; -// }; -// -//// struct data_array_t *da=(struct data_array_t *)buf; -// const struct data_t *array = (struct data_t *)buf; -// const int len = size/sizeof(struct data_t); -// int idx=0; -// -// for (idx=0;idxpeer.transport_id = NNTI_TRANSPORT_IB; \ -(p)->peer.NNTI_remote_process_t_u.ib.addr = 0; \ -(p)->peer.NNTI_remote_process_t_u.ib.port = 0; \ -(p)->peer.NNTI_remote_process_t_u.ib.qpn = 0; - - -NNTI_result_t NNTI_ib_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_ib_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -NNTI_result_t NNTI_ib_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_ib_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_ib_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ib_free ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ib_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ib_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ib_unregister_memory ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ib_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context); - -NNTI_result_t NNTI_ib_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value); - -NNTI_result_t NNTI_ib_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_clear_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_destroy_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_cancel ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ib_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -NNTI_result_t NNTI_ib_interrupt ( - const NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_ib_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status); - -NNTI_result_t NNTI_ib_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -NNTI_result_t NNTI_ib_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status); - -NNTI_result_t NNTI_ib_fini ( - const NNTI_transport_t *trans_hdl); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_IB_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/nnti_internal.h b/packages/trios/libraries/nessie/nnti/nnti_internal.h deleted file mode 100644 index 2f2be3779d09..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_internal.h +++ /dev/null @@ -1,267 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * nnti_internal.h - * - * Created on: Feb 3, 2011 - * Author: thkorde - */ - -#ifndef NNTI_INTERNAL_H_ -#define NNTI_INTERNAL_H_ - -#include "Trios_logger.h" - -#include "Trios_nnti.h" -#include - -typedef NNTI_result_t (*NNTI_INIT_FN) ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -typedef NNTI_result_t (*NNTI_GET_URL_FN)( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -typedef NNTI_result_t (*NNTI_CONNECT_FN) ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -typedef NNTI_result_t (*NNTI_DISCONNECT_FN) ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -typedef NNTI_result_t (*NNTI_ALLOC_FN) ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -typedef NNTI_result_t (*NNTI_FREE_FN) ( - NNTI_buffer_t *reg_buf); - -typedef NNTI_result_t (*NNTI_REGISTER_MEMORY_FN) ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -typedef NNTI_result_t (*NNTI_REGISTER_SEGMENTS_FN) ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -typedef NNTI_result_t (*NNTI_UNREGISTER_MEMORY_FN) ( - NNTI_buffer_t *reg_buf); - -typedef NNTI_result_t (*NNTI_SEND_FN) ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_PUT_FN) ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_GET_FN) ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_SCATTER_FN) ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_GATHER_FN) ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_ATOMIC_SET_CALLBACK_FN) ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context); - -typedef NNTI_result_t (*NNTI_ATOMIC_READ_FN) ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value); - -typedef NNTI_result_t (*NNTI_ATOMIC_FOP_FN) ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_ATOMIC_CSWAP_FN) ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_CREATE_WORK_REQUEST_FN) ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_CLEAR_WORK_REQUEST_FN) ( - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_DESTROY_WORK_REQUEST_FN) ( - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_CANCEL_FN) ( - NNTI_work_request_t *wr); - -typedef NNTI_result_t (*NNTI_CANCELALL_FN) ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -typedef NNTI_result_t (*NNTI_INTERRUPT_FN) ( - const NNTI_transport_t *trans_hdl); - -typedef NNTI_result_t (*NNTI_WAIT_FN) ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status); - -typedef NNTI_result_t (*NNTI_WAITANY_FN) ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -typedef NNTI_result_t (*NNTI_WAITALL_FN) ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status); - -typedef NNTI_result_t (*NNTI_FINI_FN) ( - const NNTI_transport_t *trans_hdl); - -typedef struct NNTI_transport_ops_t -{ - NNTI_INIT_FN nnti_init_fn; - NNTI_GET_URL_FN nnti_get_url_fn; - NNTI_CONNECT_FN nnti_connect_fn; - NNTI_DISCONNECT_FN nnti_disconnect_fn; - NNTI_ALLOC_FN nnti_alloc_fn; - NNTI_FREE_FN nnti_free_fn; - NNTI_REGISTER_MEMORY_FN nnti_register_memory_fn; - NNTI_REGISTER_SEGMENTS_FN nnti_register_segments_fn; - NNTI_UNREGISTER_MEMORY_FN nnti_unregister_memory_fn; - NNTI_SEND_FN nnti_send_fn; - NNTI_PUT_FN nnti_put_fn; - NNTI_GET_FN nnti_get_fn; - NNTI_SCATTER_FN nnti_scatter_fn; - NNTI_GATHER_FN nnti_gather_fn; - NNTI_ATOMIC_SET_CALLBACK_FN nnti_atomic_set_callback_fn; - NNTI_ATOMIC_READ_FN nnti_atomic_read_fn; - NNTI_ATOMIC_FOP_FN nnti_atomic_fop_fn; - NNTI_ATOMIC_CSWAP_FN nnti_atomic_cswap_fn; - NNTI_CREATE_WORK_REQUEST_FN nnti_create_work_request_fn; - NNTI_CLEAR_WORK_REQUEST_FN nnti_clear_work_request_fn; - NNTI_DESTROY_WORK_REQUEST_FN nnti_destroy_work_request_fn; - NNTI_CANCEL_FN nnti_cancel_fn; - NNTI_CANCELALL_FN nnti_cancelall_fn; - NNTI_INTERRUPT_FN nnti_interrupt_fn; - NNTI_WAIT_FN nnti_wait_fn; - NNTI_WAITANY_FN nnti_waitany_fn; - NNTI_WAITALL_FN nnti_waitall_fn; - NNTI_FINI_FN nnti_fini_fn; -} NNTI_transport_ops_t; - - -/** - * @brief The internal representation of a configured transport. - */ -typedef struct { - /** @brief The transport id. */ - NNTI_transport_id_t id; - - /** @brief A reference to my process that can be sent to a peer so the peer can contact me. */ - NNTI_peer_t me; - - /** @brief The transport ops. */ - NNTI_transport_ops_t ops; - - /** @brief Is the transport initialized? */ - uint8_t initialized; -} NNTI_internal_transport_t; - - -extern log_level nnti_debug_level; - - -#endif /* NNTI_INTERNAL_H_ */ diff --git a/packages/trios/libraries/nessie/nnti/nnti_mpi.cpp b/packages/trios/libraries/nessie/nnti/nnti_mpi.cpp deleted file mode 100644 index 0faabaa9d845..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_mpi.cpp +++ /dev/null @@ -1,3133 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_ptls.c - * - * Created on: Jan 13, 2011 - * Author: thkorde - */ - -#include "Trios_config.h" -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_nnti_fprint_types.h" - -// MPI is only used to increment the PID -#ifdef HAVE_TRIOS_MPI -#include -#endif - -#include -#include - -#include -#include -#include - -#include "nnti_mpi.h" -#include "nnti_utils.h" - - - -/* if defined, the RDMA initiator will send an ACK message to the RDMA - * target when the RDMA op is complete. the target process must wait - * on the target buffer in order to get the ACK. this creates two-sided - * semantics for RDMA ops. in this mode, when the wait returns the - * the RDMA op is complete and status indicates what data was addressed. - */ -#define USE_RDMA_TARGET_ACK -/* if undefined, the ACK message is NOT sent to the RDMA target when - * the RDMA op is complete. this creates one-sided semantics for RDMA - * ops. in this mode, the target has no idea when the RDMA op is - * complete and what data was addressed. NNTI_wait() returns NNTI_EINVAL - * if passed a target buffer. - */ -#undef USE_RDMA_TARGET_ACK - - -typedef struct { - - uint32_t min_atomics_vars; - -} nnti_mpi_config; - - -#define NNTI_MPI_REQUEST_TAG 0x01 -#define NNTI_MPI_ATOMICS_REQUEST_TAG 0x02 -#define NNTI_MPI_ATOMICS_RESULT_TAG 0x03 - - -#define MPI_OP_PUT_INITIATOR 1 -#define MPI_OP_GET_INITIATOR 2 -#define MPI_OP_PUT_TARGET 3 -#define MPI_OP_GET_TARGET 4 -#define MPI_OP_SEND_REQUEST 5 -#define MPI_OP_SEND_BUFFER 6 -#define MPI_OP_NEW_REQUEST 7 -#define MPI_OP_FETCH_ADD 8 -#define MPI_OP_COMPARE_SWAP 9 - - -typedef enum { - BUFFER_INIT=0, - SEND_COMPLETE=1, - RECV_COMPLETE, - RDMA_WRITE_INIT, - RDMA_RTS_COMPLETE, - RDMA_WRITE_COMPLETE, - RDMA_READ_INIT, - RDMA_RTR_COMPLETE, - RDMA_READ_COMPLETE -} mpi_op_state_t; - -typedef struct { - uint64_t offset; - uint64_t length; - int32_t tag; - uint8_t op; -} mpi_command_msg; - -typedef enum { - MPI_ATOMIC_FETCH_ADD =1, - MPI_ATOMIC_CMP_AND_SWP=2 -} mpi_atomic_op_t; - -typedef struct { - int64_t compare_add; - int64_t swap; - uint32_t index; - uint8_t op; -} mpi_atomic_request_msg; - -typedef struct { - int64_t result; -} mpi_atomic_result_msg; - -#define RDMA_CMD_INDEX 0 -#define SEND_INDEX 1 -#define GET_RECV_INDEX 2 -#define GET_SEND_INDEX 3 -#define PUT_RECV_INDEX 4 -#define PUT_SEND_INDEX 5 -#define RECV_INDEX 6 -#define ATOMICS_SEND_INDEX 7 -#define ATOMICS_RECV_INDEX 8 - -#define MAX_INDEX 9 - -#define RDMA_CMD_REQUEST_ACTIVE 0x001 -#define SEND_REQUEST_ACTIVE 0x002 -#define GET_RECV_REQUEST_ACTIVE 0x004 -#define GET_SEND_REQUEST_ACTIVE 0x008 -#define PUT_RECV_REQUEST_ACTIVE 0x010 -#define PUT_SEND_REQUEST_ACTIVE 0x020 -#define RECV_REQUEST_ACTIVE 0x040 -#define ATOMICS_SEND_REQUEST_ACTIVE 0x080 -#define ATOMICS_RECV_REQUEST_ACTIVE 0x100 - -typedef struct mpi_work_request { - NNTI_work_request_t *nnti_wr; - - NNTI_buffer_t *reg_buf; - NNTI_peer_t peer; - uint64_t src_offset; - uint64_t dst_offset; - uint64_t length; - int32_t tag; - MPI_Request request[MAX_INDEX]; - MPI_Request *request_ptr; - uint32_t request_count; - int request_index; - uint8_t active_requests; - - mpi_command_msg cmd_msg; - - int atomics_result_index; - - mpi_op_state_t op_state; - - MPI_Status last_event; - uint8_t last_op; - uint8_t is_last_op_complete; -} mpi_work_request; - -typedef std::deque wr_queue_t; -typedef std::deque::iterator wr_queue_iter_t; - -typedef struct mpi_memory_handle { - int64_t cmd_tag; - int64_t get_data_tag; - int64_t put_data_tag; - - wr_queue_t wr_queue; - nthread_lock_t wr_queue_lock; -} mpi_memory_handle; - - -#define NUM_REQ_QUEUES 2 -typedef struct mpi_request_queue_handle { - NNTI_buffer_t *reg_buf; - - /* incoming queue */ - char *req_queue; - - /* each message is no larger than req_size */ - int req_size; - - /* number of requests slots in the queue */ - int req_count; - - MPI_Request *mpi_request_list; - uint64_t last_request_index; - -} mpi_request_queue_handle; - -typedef struct { - nthread_lock_t lock; - int64_t value; -} mpi_atomic_t; - -typedef struct mpi_transport_global { - - int rank; - int size; - char proc_name[MPI_MAX_PROCESSOR_NAME]; - - MPI_Comm nnti_comm; - - nthread_counter_t mbits; - - mpi_request_queue_handle req_queue; - - mpi_atomic_t *atomics; - mpi_atomic_request_msg atomics_request_msg; - mpi_atomic_result_msg atomics_result_msg; - MPI_Request atomics_recv_request; - MPI_Request atomics_send_request; - - bool init_called_mpi_init; - -} mpi_transport_global; - - - -static nthread_lock_t nnti_mpi_lock; - - -static int process_event( - mpi_work_request *mpi_wr, - const MPI_Status *event); -static NNTI_result_t setup_atomics(void); -static int check_atomic_operation(void); -static int check_target_buffer_progress(void); -static NNTI_result_t post_atomics_recv_request(void); -static NNTI_result_t post_recv_queue_work_request( - NNTI_buffer_t *reg_buf, - int64_t tag, - uint64_t length, - uint64_t count, - MPI_Request *request_list); -static NNTI_result_t post_rdma_target_work_request( - NNTI_buffer_t *reg_buf); -static NNTI_result_t repost_recv_work_request( - mpi_work_request *mpi_wr); -static NNTI_result_t repost_rdma_target_work_request( - mpi_work_request *mpi_wr); -static int is_wr_complete( - mpi_work_request *mpi_wr); -static int8_t is_wr_complete( - NNTI_work_request_t *wr); -static int8_t is_any_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - uint32_t *which); -static int8_t is_all_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -static NNTI_result_t insert_target_buffer(NNTI_buffer_t *buf); -static NNTI_buffer_t *del_target_buffer(NNTI_buffer_t *buf); -//static void print_target_buffer_deque(); - -static void create_status( - NNTI_work_request_t *wr, - mpi_work_request *mpi_wr, - int nnti_rc, - NNTI_status_t *status); -static void create_peer( - NNTI_peer_t *peer, - int rank); - -static void config_init( - nnti_mpi_config *c); -static void config_get_from_env( - nnti_mpi_config *c); - - -#define MPI_MEM_HDL(b) ((mpi_memory_handle *)((b)->transport_private)) -#define MPI_WORK_REQUEST(wr) ((mpi_work_request *)((wr)->transport_private)) - - -static std::map buffers_by_bufhash; -typedef std::map::iterator buf_by_bufhash_iter_t; -typedef std::pair buf_by_bufhash_t; -static nthread_lock_t nnti_buf_bufhash_lock; - -static std::map wr_by_wrhash; -typedef std::map::iterator wr_by_wrhash_iter_t; -typedef std::pair wr_by_wrhash_t; -static nthread_lock_t nnti_wr_wrhash_lock; - -typedef std::deque target_buffer_queue_t; -typedef std::deque::iterator target_buffer_queue_iter_t; -static nthread_lock_t nnti_target_buffer_queue_lock; - -target_buffer_queue_t target_buffers; - - -static nnti_mpi_config config; - - -static mpi_transport_global transport_global_data; -static const int MAX_SLEEP = 10; /* in milliseconds */ - -/** - * @brief Initialize NNTI to use a specific transport. - * - * Enable the use of a particular transport by this process. my_url - * allows the process to have some control (if possible) over the - * URL assigned for the transport. For example, a Portals URL to put - * might be "ptl://-1,128". This would tell Portals to use the default - * network ID, but use PID=128. If the transport - * can be initialized without this info (eg. a Portals client), my_url can - * be NULL or empty. - */ -NNTI_result_t NNTI_mpi_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - static uint8_t initialized=FALSE; - - int mpi_initialized=FALSE; - int name_len=0; - - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - - - if (!initialized) { - - nthread_lock_init(&nnti_mpi_lock); - nthread_lock_init(&nnti_buf_bufhash_lock); - nthread_lock_init(&nnti_wr_wrhash_lock); - nthread_lock_init(&nnti_target_buffer_queue_lock); - - config_init(&config); - config_get_from_env(&config); - - if (my_url != NULL) { - log_error(nnti_debug_level,"The MPI transport does not accept a URL at init. Ignoring URL."); - } - - log_debug(nnti_debug_level, "initializing MPI transport"); - - memset(&transport_global_data, 0, sizeof(mpi_transport_global)); - - rc=MPI_Initialized(&mpi_initialized); - if (rc) { - log_fatal(nnti_debug_level,"MPI_Initialized() failed, %d", rc); - abort(); - } - - if (mpi_initialized==FALSE) { - int argc=0; - char **argv=NULL; - int provided=-1; - - log_debug(nnti_debug_level, "initializing MPI library"); - - rc=MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &provided); - if (rc) { - log_fatal(nnti_debug_level,"MPI_Init_thread() failed, %d", rc); - abort(); - } - - transport_global_data.init_called_mpi_init=true; - } - -// MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN); - MPI_Comm_size(MPI_COMM_WORLD, &transport_global_data.size); - MPI_Comm_rank(MPI_COMM_WORLD, &transport_global_data.rank); - MPI_Get_processor_name(transport_global_data.proc_name, &name_len); - - if (logging_info(nnti_debug_level)) { - fprintf(logger_get_file(), "MPI Initialized: rank=%llu, size=%llu, proc_name=%s\n", - (unsigned long long)transport_global_data.rank, - (unsigned long long)transport_global_data.size, - transport_global_data.proc_name); - } - - nthread_counter_init(&transport_global_data.mbits); - nthread_counter_set(&transport_global_data.mbits, 0x111); - - setup_atomics(); - - create_peer(&trans_hdl->me, transport_global_data.rank); - - initialized = TRUE; - } - - - log_debug(nnti_debug_level, "exit"); - - - return(nnti_rc); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recognize. - * - * URL format: "transport://address/memory_descriptor" - * - transport - (required) identifies how the URL should parsed - * - address - (required) uniquely identifies a location on the network - * - ex. "ptl://nid:pid/", "ib://ip_addr:port" - * - memory_descriptor - (optional) transport-specific representation of RMA params - */ -NNTI_result_t NNTI_mpi_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - assert(trans_hdl); - assert(url); - assert(maxlen>0); - - strncpy(url, trans_hdl->me.url, maxlen); - url[maxlen-1]='\0'; - - return(nnti_rc); -} - - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * -// * If the peer is found and responds -// * to a ping, a handle will be allocated and assigned to the pointer. This -// * handle should be used to move data to/from the peer. - * - * Connectionless transport: parse and populate - * Connected transport: parse, connection and populate - * - */ -NNTI_result_t NNTI_mpi_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; - - int peer_rank; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(peer_hdl); - - if (url != NULL) { - if ((nnti_rc=nnti_url_get_transport(url, transport, NNTI_URL_LEN)) != NNTI_OK) { - return(nnti_rc); - } - if (0!=strcmp(transport, "mpi")) { - /* the peer described by 'url' is not an MPI peer */ - return(NNTI_EINVAL); - } - - if ((nnti_rc=nnti_url_get_address(url, address, NNTI_URL_LEN)) != NNTI_OK) { - return(nnti_rc); - } - - peer_rank=strtol(address, NULL, 0); - } else { - /* */ - return(NNTI_EINVAL); - } - - create_peer( - peer_hdl, - peer_rank); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - */ -NNTI_result_t NNTI_mpi_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - assert(trans_hdl); - assert(peer_hdl); - - return(nnti_rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_mpi_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - char *buf=(char *)malloc(element_size*num_elements); - assert(buf); - - nnti_rc=NNTI_mpi_register_memory( - trans_hdl, - buf, - element_size, - num_elements, - ops, - reg_buf); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_mpi_alloc", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_mpi_free ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - char *buf=NNTI_BUFFER_C_POINTER(reg_buf); - assert(buf); - - nnti_rc=NNTI_mpi_unregister_memory(reg_buf); - - free(buf); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_mpi_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ -// int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(buffer); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - mpi_mem_hdl=new mpi_memory_handle(); - assert(mpi_mem_hdl); - nthread_lock_init(&mpi_mem_hdl->wr_queue_lock); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size = element_size; - reg_buf->payload = (uint64_t)buffer; - reg_buf->transport_private = (uint64_t)mpi_mem_hdl; - - log_debug(nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - - if (ops == NNTI_BOP_RECV_QUEUE) { - mpi_mem_hdl->cmd_tag = 0; - mpi_mem_hdl->get_data_tag = 0; - mpi_mem_hdl->put_data_tag = NNTI_MPI_REQUEST_TAG; - } else { - mpi_mem_hdl->cmd_tag = nthread_counter_increment(&transport_global_data.mbits); - mpi_mem_hdl->get_data_tag = nthread_counter_increment(&transport_global_data.mbits); - mpi_mem_hdl->put_data_tag = nthread_counter_increment(&transport_global_data.mbits); - } - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(1, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=1; - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id = NNTI_TRANSPORT_MPI; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.size = element_size; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag = mpi_mem_hdl->cmd_tag; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.get_data_tag = mpi_mem_hdl->get_data_tag; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.put_data_tag = mpi_mem_hdl->put_data_tag; - - if (ops == NNTI_BOP_RECV_QUEUE) { - mpi_request_queue_handle *q_hdl=&transport_global_data.req_queue; - - q_hdl->reg_buf=reg_buf; - - q_hdl->req_queue =buffer; - q_hdl->req_size =element_size; - q_hdl->req_count =num_elements; - q_hdl->last_request_index=0; - q_hdl->mpi_request_list =(MPI_Request*)calloc(q_hdl->req_count, sizeof(MPI_Request)); - - /* initialize the buffer */ - memset(q_hdl->req_queue, 0, q_hdl->req_count*q_hdl->req_size); - - post_recv_queue_work_request( - reg_buf, - NNTI_MPI_REQUEST_TAG, - q_hdl->req_size, - q_hdl->req_count, - q_hdl->mpi_request_list); - - } else if ((ops & NNTI_BOP_REMOTE_READ) || (ops & NNTI_BOP_REMOTE_WRITE)) { - post_rdma_target_work_request( - reg_buf); - insert_target_buffer( - reg_buf); - - } - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_mpi_register_memory", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Prepare a list of memory segments for network operations. - * - * Wrap a list of user allocated memory segments in an NNTI_buffer_t. The - * transport may take additional actions to prepare the memory segments for - * network send/receive. If the memory segments don't meet the transport's - * requirements for memory regions, then errors or poor performance may - * result. - * - */ -NNTI_result_t NNTI_mpi_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - return NNTI_ENOTSUP; -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_mpi_unregister_memory ( - NNTI_buffer_t *reg_buf) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - mpi_memory_handle *mpi_mem_hdl=NULL; - log_level debug_level = nnti_debug_level; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - log_debug(nnti_debug_level, "unregistering reg_buf(%p) buf(%p)", reg_buf, reg_buf->payload); - - del_target_buffer(reg_buf); - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - while (!mpi_mem_hdl->wr_queue.empty()) { - mpi_work_request *mpi_wr=NULL; - - mpi_wr=mpi_mem_hdl->wr_queue.front(); - mpi_mem_hdl->wr_queue.pop_front(); - - log_debug(nnti_debug_level, "removing pending mpi_wr=%p mpi_wr->active_requests=%X", mpi_wr, (uint32_t)mpi_wr->active_requests); - -// for (int i=0;iactive_requests & (1<request[%d]=%p", i, &mpi_wr->request[i]); -// nthread_lock(&nnti_mpi_lock); -// MPI_Cancel(&mpi_wr->request[i]); -// nthread_unlock(&nnti_mpi_lock); -// } -// } - -// del_wr_wrhash(mpi_wr); -// if ((mpi_mem_hdl->type!=REQUEST_BUFFER) && -// (mpi_mem_hdl->type!=RECEIVE_BUFFER)) { -// free(mpi_wr); -// } - } - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - if (mpi_mem_hdl) - nthread_lock_fini(&mpi_mem_hdl->wr_queue_lock); - delete mpi_mem_hdl; - if (reg_buf->buffer_segments.NNTI_remote_addr_array_t_val) - free(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val); - - reg_buf->transport_id = NNTI_TRANSPORT_NULL; - MPI_SET_MATCH_ANY(®_buf->buffer_owner); - reg_buf->ops = (NNTI_buf_ops_t)0; - reg_buf->payload_size = 0; - reg_buf->payload = 0; - reg_buf->transport_private = 0; - - log_debug(debug_level, "Finished unregistering, rc=%d",rc); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - */ -NNTI_result_t NNTI_mpi_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - mpi_memory_handle *mpi_mem_hdl=NULL; - mpi_work_request *mpi_wr=NULL; - int dest_rank; - uint32_t tag; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - assert(msg_hdl); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "msg_hdl", - "NNTI_mpi_send", msg_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_hdl", - "NNTI_mpi_send", dest_hdl); - } - - if ((dest_hdl == NULL) || (dest_hdl->ops == NNTI_BOP_RECV_QUEUE)) { - mpi_mem_hdl=MPI_MEM_HDL(msg_hdl); - assert(mpi_mem_hdl); - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - - mpi_wr->nnti_wr =wr; - mpi_wr->reg_buf =(NNTI_buffer_t *)msg_hdl; - mpi_wr->src_offset=0; - mpi_wr->dst_offset=0; - mpi_wr->length =msg_hdl->payload_size; - mpi_wr->op_state =BUFFER_INIT; - - mpi_wr->peer =*peer_hdl; - mpi_wr->last_op=MPI_OP_SEND_REQUEST; - - dest_rank=peer_hdl->peer.NNTI_remote_process_t_u.mpi.rank; - tag =NNTI_MPI_REQUEST_TAG; - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Issend( - (char*)msg_hdl->payload, - msg_hdl->payload_size, - MPI_BYTE, - dest_rank, - tag, - MPI_COMM_WORLD, - &mpi_wr->request[SEND_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to send with Isend"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - mpi_wr->request_ptr =&mpi_wr->request[SEND_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= SEND_REQUEST_ACTIVE; - - wr->transport_id =msg_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)msg_hdl; - wr->ops =NNTI_BOP_LOCAL_READ; - wr->transport_private=(uint64_t)mpi_wr; - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "sending to (rank=%d, tag=%d)", dest_rank, tag); - - } else { - NNTI_mpi_put(msg_hdl, 0, msg_hdl->payload_size, dest_hdl, 0, wr); - } - -cleanup: - log_debug(nnti_debug_level, "exit (wr=%p ; mpi_wr=%p ; last_op=%d)", wr, (mpi_work_request*)wr->transport_private, ((mpi_work_request*)wr->transport_private)->last_op); - - return(nnti_rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_mpi_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - mpi_memory_handle *mpi_mem_hdl=NULL; - mpi_work_request *mpi_wr=NULL; - int dest_rank; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "src_buffer_hdl", - "NNTI_mpi_put", src_buffer_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_buffer_hdl", - "NNTI_mpi_put", dest_buffer_hdl); - } - - mpi_mem_hdl=MPI_MEM_HDL(src_buffer_hdl); - assert(mpi_mem_hdl); - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - - mpi_wr->nnti_wr =wr; - mpi_wr->reg_buf =(NNTI_buffer_t *)src_buffer_hdl; - mpi_wr->peer =dest_buffer_hdl->buffer_owner; - mpi_wr->src_offset=src_offset; - mpi_wr->dst_offset=dest_offset; - mpi_wr->length =src_length; - mpi_wr->op_state =RDMA_WRITE_INIT; - mpi_wr->last_op =MPI_OP_PUT_INITIATOR; - - dest_rank=dest_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.mpi.rank; - - mpi_wr->cmd_msg.length=src_length; - mpi_wr->cmd_msg.offset=dest_offset; - mpi_wr->cmd_msg.tag =dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.put_data_tag; - mpi_wr->cmd_msg.op =MPI_OP_PUT_TARGET; - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Issend( - &mpi_wr->cmd_msg, - sizeof(mpi_wr->cmd_msg), - MPI_BYTE, - dest_rank, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag, - MPI_COMM_WORLD, - &mpi_wr->request[RDMA_CMD_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to Issend CMD msg"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Issend( - (char*)src_buffer_hdl->payload+src_offset, - src_length, - MPI_BYTE, - dest_rank, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.put_data_tag, - MPI_COMM_WORLD, - &mpi_wr->request[PUT_SEND_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to Issend region"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - mpi_wr->request_ptr=&mpi_wr->request[RDMA_CMD_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= RDMA_CMD_REQUEST_ACTIVE; - mpi_wr->active_requests |= PUT_SEND_REQUEST_ACTIVE; - - log_debug(nnti_debug_level, "putting to (%s, dest_rank=%d, cmd_tag=%d, put_data_tag=%d)", - dest_buffer_hdl->buffer_owner.url, dest_rank, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.put_data_tag); - - wr->transport_id =src_buffer_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)src_buffer_hdl; - wr->ops =NNTI_BOP_LOCAL_READ; - wr->transport_private=(uint64_t)mpi_wr; - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - -cleanup: - log_debug(nnti_debug_level, "exit (wr=%p ; mpi_wr=%p ; last_op=%d)", wr, (mpi_work_request*)wr->transport_private, ((mpi_work_request*)wr->transport_private)->last_op); - - return(nnti_rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_mpi_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - mpi_memory_handle *mpi_mem_hdl=NULL; - mpi_work_request *mpi_wr=NULL; - int src_rank; - - log_debug(nnti_debug_level, "enter"); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "src_buffer_hdl", - "NNTI_mpi_get", src_buffer_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_buffer_hdl", - "NNTI_mpi_get", dest_buffer_hdl); - } - - log_debug(nnti_debug_level, "getting from (%s, src_offset=%llu, src_length=%llu, dest_offset=%llu)", - src_buffer_hdl->buffer_owner.url, src_offset, src_length, dest_offset); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "src_buffer_hdl", - "NNTI_mpi_get", src_buffer_hdl); - fprint_NNTI_buffer(logger_get_file(), "dest_buffer_hdl", - "NNTI_mpi_get", dest_buffer_hdl); - } - - mpi_mem_hdl=MPI_MEM_HDL(dest_buffer_hdl); - assert(mpi_mem_hdl); - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - - mpi_wr->nnti_wr =wr; - mpi_wr->reg_buf =(NNTI_buffer_t *)dest_buffer_hdl; - mpi_wr->peer =src_buffer_hdl->buffer_owner; - mpi_wr->src_offset=src_offset; - mpi_wr->dst_offset=dest_offset; - mpi_wr->length =src_length; - mpi_wr->op_state =RDMA_READ_INIT; - mpi_wr->last_op =MPI_OP_GET_INITIATOR; - - src_rank=src_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.mpi.rank; - - mpi_wr->cmd_msg.length=src_length; - mpi_wr->cmd_msg.offset=src_offset; - mpi_wr->cmd_msg.tag=dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.get_data_tag; - mpi_wr->cmd_msg.op =MPI_OP_GET_TARGET; - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Irecv( - (char*)dest_buffer_hdl->payload+dest_offset, - src_length, - MPI_BYTE, - src_rank, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.get_data_tag, - MPI_COMM_WORLD, - &mpi_wr->request[GET_RECV_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to Irecv region"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Issend( - &mpi_wr->cmd_msg, - sizeof(mpi_wr->cmd_msg), - MPI_BYTE, - src_rank, - src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag, - MPI_COMM_WORLD, - &mpi_wr->request[RDMA_CMD_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to Issend CMD msg"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - mpi_wr->request_ptr=&mpi_wr->request[RDMA_CMD_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= RDMA_CMD_REQUEST_ACTIVE; - mpi_wr->active_requests |= GET_RECV_REQUEST_ACTIVE; - - log_debug(nnti_debug_level, "getting from (%s, src_rank=%d, cmd_tag=%d, get_data_tag=%d)", - dest_buffer_hdl->buffer_owner.url, src_rank, - src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.get_data_tag); - - wr->transport_id =dest_buffer_hdl->transport_id; - wr->reg_buf =(NNTI_buffer_t*)dest_buffer_hdl; - wr->ops =NNTI_BOP_LOCAL_WRITE; - wr->transport_private=(uint64_t)mpi_wr; - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - -cleanup: - log_debug(nnti_debug_level, "exit (wr=%p ; mpi_wr=%p ; last_op=%d)", wr, (mpi_work_request*)wr->transport_private, ((mpi_work_request*)wr->transport_private)->last_op); - - return(nnti_rc); -} - - -/** - * @brief Transfer data to a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to put. - * \param[in] src_length The number of bytes to put. - * \param[in] dest_buffer_list A list of buffers to put the data into. - * \param[in] dest_count The number of destination buffers. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_mpi_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr) -{ - return NNTI_ENOTSUP; -} - - -/** - * @brief Transfer data from a peer. - * - * \param[in] src_buffer_list A list of buffers containing the data to get. - * \param[in] src_length The number of bytes to get. - * \param[in] src_count The number of source buffers. - * \param[in] dest_buffer_hdl A buffer to get the data into. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_mpi_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_mpi_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_mpi_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value) -{ - nthread_lock(&transport_global_data.atomics[local_atomic].lock); - *value = transport_global_data.atomics[local_atomic].value; - nthread_unlock(&transport_global_data.atomics[local_atomic].lock); - - return NNTI_OK; -} - - -NNTI_result_t NNTI_mpi_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - mpi_work_request *mpi_wr=NULL; - int dest_rank; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - - mpi_wr->nnti_wr =wr; - mpi_wr->op_state =BUFFER_INIT; - - mpi_wr->atomics_result_index=result_atomic; - - mpi_wr->peer =*peer_hdl; - mpi_wr->last_op=MPI_OP_FETCH_ADD; - dest_rank =peer_hdl->peer.NNTI_remote_process_t_u.mpi.rank; - - transport_global_data.atomics_request_msg.op =MPI_ATOMIC_FETCH_ADD; - transport_global_data.atomics_request_msg.index =target_atomic; - transport_global_data.atomics_request_msg.compare_add=operand; - - log_debug(nnti_debug_level, "sending fetch-add to (rank=%d)", dest_rank); - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Isend( - (char*)&transport_global_data.atomics_request_msg, - sizeof(transport_global_data.atomics_request_msg), - MPI_BYTE, - dest_rank, - NNTI_MPI_ATOMICS_REQUEST_TAG, - MPI_COMM_WORLD, - &mpi_wr->request[ATOMICS_SEND_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to send with Isend"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Irecv( - (char*)&transport_global_data.atomics_result_msg, - sizeof(transport_global_data.atomics_result_msg), - MPI_BYTE, - dest_rank, - NNTI_MPI_ATOMICS_RESULT_TAG, - MPI_COMM_WORLD, - &mpi_wr->request[ATOMICS_RECV_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to post recv with Irecv"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - mpi_wr->request_ptr =&mpi_wr->request[ATOMICS_SEND_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= ATOMICS_SEND_REQUEST_ACTIVE; - mpi_wr->active_requests |= ATOMICS_RECV_REQUEST_ACTIVE; - - wr->transport_id =trans_hdl->id; - wr->reg_buf =(NNTI_buffer_t*)NULL; - wr->ops =NNTI_BOP_ATOMICS; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)mpi_wr; - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -NNTI_result_t NNTI_mpi_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - mpi_work_request *mpi_wr=NULL; - int dest_rank; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - - mpi_wr->nnti_wr =wr; - mpi_wr->op_state =BUFFER_INIT; - - mpi_wr->atomics_result_index=result_atomic; - - mpi_wr->peer =*peer_hdl; - mpi_wr->last_op=MPI_OP_FETCH_ADD; - dest_rank =peer_hdl->peer.NNTI_remote_process_t_u.mpi.rank; - - transport_global_data.atomics_request_msg.op =MPI_ATOMIC_CMP_AND_SWP; - transport_global_data.atomics_request_msg.index =target_atomic; - transport_global_data.atomics_request_msg.compare_add=compare_operand; - transport_global_data.atomics_request_msg.swap =swap_operand; - - log_debug(nnti_debug_level, "sending compare-swap to (rank=%d)", dest_rank); - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Isend( - (char*)&transport_global_data.atomics_request_msg, - sizeof(transport_global_data.atomics_request_msg), - MPI_BYTE, - dest_rank, - NNTI_MPI_ATOMICS_REQUEST_TAG, - MPI_COMM_WORLD, - &mpi_wr->request[ATOMICS_SEND_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to send with Isend"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Irecv( - (char*)&transport_global_data.atomics_result_msg, - sizeof(transport_global_data.atomics_result_msg), - MPI_BYTE, - dest_rank, - NNTI_MPI_ATOMICS_RESULT_TAG, - MPI_COMM_WORLD, - &mpi_wr->request[ATOMICS_RECV_INDEX]); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to post recv with Irecv"); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - mpi_wr->request_ptr =&mpi_wr->request[ATOMICS_SEND_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= ATOMICS_SEND_REQUEST_ACTIVE; - mpi_wr->active_requests |= ATOMICS_RECV_REQUEST_ACTIVE; - - wr->transport_id =trans_hdl->id; - wr->reg_buf =(NNTI_buffer_t*)NULL; - wr->ops =NNTI_BOP_ATOMICS; - wr->result =NNTI_OK; - wr->transport_private=(uint64_t)mpi_wr; - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Create a receive work request that can be used to wait for buffer - * operations to complete. - * - */ -NNTI_result_t NNTI_mpi_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr) -{ - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p ; wr=%p)", reg_buf, wr); - - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - wr->transport_id =reg_buf->transport_id; - wr->reg_buf =reg_buf; - wr->ops =reg_buf->ops; - wr->transport_private=(uint64_t)NULL; - - log_debug(nnti_debug_level, "exit (reg_buf=%p ; wr=%p)", reg_buf, wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from a previous receive - * and prepares it for reuse. - * - */ -NNTI_result_t NNTI_mpi_clear_work_request ( - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - wr->transport_private=(uint64_t)NULL; - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from reg_buf. - * - */ -NNTI_result_t NNTI_mpi_destroy_work_request ( - NNTI_work_request_t *wr) -{ - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - wr->transport_id =NNTI_TRANSPORT_NULL; - wr->reg_buf =NULL; - wr->ops =(NNTI_buf_ops_t)0; - wr->transport_private=(uint64_t)NULL; - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Attempts to cancel an NNTI opertion. - * - */ -NNTI_result_t NNTI_mpi_cancel ( - NNTI_work_request_t *wr) -{ - return NNTI_ENOTSUP; -} - - -/** - * @brief Attempts to cancel a list of NNTI opertions. - * - */ -NNTI_result_t NNTI_mpi_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - return NNTI_ENOTSUP; -} - - -/** - * @brief Interrupts NNTI_wait*() - * - */ -NNTI_result_t NNTI_mpi_interrupt ( - const NNTI_transport_t *trans_hdl) -{ - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_ENOTSUP; -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t NNTI_mpi_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status) -{ - int rc=MPI_SUCCESS; - NNTI_result_t nnti_rc=NNTI_OK; - mpi_memory_handle *mpi_mem_hdl=NULL; - mpi_work_request *mpi_wr=NULL; - - long elapsed_time=0; -// long timeout_per_call; - MPI_Status event; - int done=FALSE; - - int ops_completed=0; - - log_level debug_level=nnti_debug_level; - - long entry_time=trios_get_time_ms(); - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - trios_start_timer(total_time); - - log_debug(debug_level, "enter"); - - assert(wr); - assert(status); - - mpi_wr=MPI_WORK_REQUEST(wr); - if (wr->ops != NNTI_BOP_ATOMICS) { - mpi_mem_hdl=MPI_MEM_HDL(wr->reg_buf); - assert(mpi_mem_hdl); - if (mpi_wr==NULL) { - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_wr=mpi_mem_hdl->wr_queue.front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - wr->transport_private=(uint64_t)mpi_wr; - } - } - assert(mpi_wr); - - if (is_wr_complete(mpi_wr) == TRUE) { - log_debug(debug_level, "work request already complete"); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "work request NOT complete"); - -// timeout_per_call = MIN_TIMEOUT; - - while (1) { - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - return NNTI_ECANCELED; - } - - ops_completed += check_atomic_operation(); - ops_completed += check_target_buffer_progress(); - - if (ops_completed > 0) { - ops_completed=0; - if (is_wr_complete(mpi_wr) == TRUE) { - break; - } - } - - log_debug(debug_level, "waiting on wr(%p) request(%p)", wr , mpi_wr->request_ptr); - - memset(&event, 0, sizeof(MPI_Status)); - done=FALSE; - trios_start_timer(call_time); - nthread_lock(&nnti_mpi_lock); - rc = MPI_Testany(mpi_wr->request_count, mpi_wr->request_ptr, &mpi_wr->request_index, &done, &event); - nthread_unlock(&nnti_mpi_lock); - trios_stop_timer("NNTI_mpi_wait - MPI_Test", call_time); - - log_debug(debug_level, "polling status is %d, which_req=%d, done=%d", rc, mpi_wr->request_index, done); - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\tsource = %d", event.MPI_SOURCE); - log_debug(debug_level, "\ttag = %d", event.MPI_TAG); - log_debug(debug_level, "\terror = %d", event.MPI_ERROR); - log_debug(debug_level, "}"); - - /* MPI_Testany() says no active requests. this is not fatal. */ - if ((rc==MPI_SUCCESS) && (mpi_wr->request_index==MPI_UNDEFINED) && (done==TRUE)) { - log_debug(debug_level, "MPI_Testany() says there a no active requests (rc=%d, which_req=%d, done=%d)", rc, mpi_wr->request_index, done); - } - /* MPI_Testany() says requests have completed */ - else if (rc == MPI_SUCCESS) { - /* case 1: success */ - if (done == TRUE) { - nnti_rc = NNTI_OK; - process_event(mpi_wr, &event); - } - /* case 2: timed out */ - else { - elapsed_time = (trios_get_time_ms() - entry_time); - - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout > 0) && (elapsed_time >= timeout))) { - log_debug(debug_level, "MPI_Test() timed out"); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - int timeout_remaining=timeout-elapsed_time; - if ((timeout < 0) || (timeout_remaining > MAX_SLEEP)) { - nnti_sleep(MAX_SLEEP); - } else { - if (timeout_remaining > 0) { - nnti_sleep(timeout_remaining); - } - } - - /* continue if the timeout has not expired */ - /* log_debug(debug_level, "timedout... continuing"); */ - - continue; - } - } - /* MPI_Testany() failure */ - else { - log_error(debug_level, "MPI_Testany() failed (request=%p): rc=%d", - mpi_wr->request_ptr, rc); - nnti_rc = NNTI_EIO; - break; - } - - if (is_wr_complete(mpi_wr) == TRUE) { - break; - } - } - } - - create_status(wr, mpi_wr, nnti_rc, status); - - - if (nnti_rc==NNTI_OK) { - switch (mpi_wr->last_op) { - case MPI_OP_NEW_REQUEST: - mpi_mem_hdl=MPI_MEM_HDL(mpi_wr->reg_buf); - assert(mpi_mem_hdl); - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.pop_front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - repost_recv_work_request(mpi_wr); - break; - case MPI_OP_PUT_TARGET: - case MPI_OP_GET_TARGET: - repost_rdma_target_work_request(mpi_wr); - break; - case MPI_OP_FETCH_ADD: - case MPI_OP_COMPARE_SWAP: - free(mpi_wr); - break; - default: - log_debug(nnti_debug_level, "status->offset=%llu, status->length=%llu", - (uint64_t)status->offset, (uint64_t)status->length); - uint32_t index=status->offset/status->length; - log_debug(nnti_debug_level, "status->offset=%llu, status->length=%llu, index=%llu", - (uint64_t)status->offset, (uint64_t)status->length, (uint64_t)index); - - mpi_mem_hdl=MPI_MEM_HDL(mpi_wr->reg_buf); - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - wr_queue_iter_t victim=find(mpi_mem_hdl->wr_queue.begin(), mpi_mem_hdl->wr_queue.end(), mpi_wr); - if (victim != mpi_mem_hdl->wr_queue.end()) { - log_debug(debug_level, "erasing mpi_wr(%p) from wr_queue", mpi_wr); - mpi_mem_hdl->wr_queue.erase(victim); - } - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - free(mpi_wr); - break; - } - } - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_mpi_wait", status); - } - - log_debug(debug_level, "exit"); - - trios_stop_timer("NNTI_mpi_wait", total_time); - - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on any buffer in wr_list to complete. - * - * Wait for remote_op on any buffer in wr_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in wr_list must be registered with the same transport. - * 2) You can't wait on the request queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_mpi_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status) -{ - int rc=MPI_SUCCESS; - NNTI_result_t nnti_rc=NNTI_OK; - mpi_memory_handle *mpi_mem_hdl=NULL; - mpi_work_request *mpi_wr=NULL; - - uint32_t mpi_request_count=0; - MPI_Request *mpi_requests=NULL; - uint32_t *request_to_wr_index=NULL; - MPI_Status event; - int done=FALSE; - - int ops_completed=0; - - int which_req=0; - - long elapsed_time=0; -// long timeout_per_call; - - long entry_time=trios_get_time_ms(); - - log_level debug_level=nnti_debug_level; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - trios_start_timer(total_time); - - log_debug(debug_level, "enter"); - - assert(wr_list); - assert(wr_count > 0); - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_mpi_wait(wr_list[0], timeout, status); - *which=0; - goto cleanup; - } - - for (uint32_t i=0;ireg_buf); - assert(mpi_mem_hdl); - mpi_wr=MPI_WORK_REQUEST(wr_list[i]); - if (mpi_wr==NULL) { - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_wr=mpi_mem_hdl->wr_queue.front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - wr_list[i]->transport_private=(uint64_t)mpi_wr; - } - assert(mpi_wr); - } - } - - if (is_any_wr_complete(wr_list, wr_count, which) == TRUE) { - log_debug(debug_level, "work request already complete (which=%u, wr_list[%d]=%p)", *which, *which, wr_list[*which]); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "work request NOT complete (wr_list=%p)", wr_list); - -// timeout_per_call = MIN_TIMEOUT; - - while (1) { - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - return NNTI_ECANCELED; - } - - ops_completed += check_atomic_operation(); - ops_completed += check_target_buffer_progress(); - - if (ops_completed > 0) { - ops_completed=0; - if (is_any_wr_complete(wr_list, wr_count, which) == TRUE) { - break; - } - } - - log_debug(debug_level, "waiting on wr_list(%p)", wr_list); - - /* - * The list of MPI_Requests is recreated each time through this loop. There is probably a better way. - */ - /* find the number of MPI_Requests in wr_list */ - mpi_request_count=0; - for (uint32_t i=0;irequest_count; - } - } - } - log_debug(debug_level, "wr_list contains %lu MPI_Requests", mpi_request_count); - - mpi_requests=(MPI_Request *)realloc(mpi_requests, mpi_request_count * sizeof(MPI_Request)); - assert(mpi_requests); - request_to_wr_index=(uint32_t *)realloc(request_to_wr_index, mpi_request_count * sizeof(uint32_t)); - assert(request_to_wr_index); - - uint32_t request_index=0; - /* for each NNTI work request */ - for (uint32_t i=0;irequest_count;j++) { - mpi_requests[request_index]=mpi_wr->request_ptr[j]; - request_to_wr_index[request_index]=i; - request_index++; - } - } - } - } - - memset(&event, 0, sizeof(MPI_Status)); - done=FALSE; - trios_start_timer(call_time); - nthread_lock(&nnti_mpi_lock); - rc = MPI_Testany(mpi_request_count, mpi_requests, &which_req, &done, &event); - nthread_unlock(&nnti_mpi_lock); - trios_stop_timer("NNTI_mpi_waitany - MPI_Testany", call_time); - - log_debug(debug_level, "polling status is %d, which_req=%d, done=%d", rc, which_req, done); - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\tsource = %d", event.MPI_SOURCE); - log_debug(debug_level, "\ttag = %d", event.MPI_TAG); - log_debug(debug_level, "\terror = %d", event.MPI_ERROR); - log_debug(debug_level, "}"); - - /* MPI_Testany() says no active requests. this is not fatal. */ - if ((rc==MPI_SUCCESS) && (which_req==MPI_UNDEFINED) && (done==TRUE)) { - log_debug(debug_level, "MPI_Testany() says there a no active requests (rc=%d, which_req=%d, done=%d)", rc, which_req, done); - } - /* MPI_Testany() says requests have completed */ - else if (rc == MPI_SUCCESS) { - /* case 1: success */ - if (done == TRUE) { - *which=request_to_wr_index[which_req]; - log_debug(debug_level, "*which == %d", *which); - nnti_rc = NNTI_OK; - process_event(MPI_WORK_REQUEST(wr_list[*which]), &event); - } - /* case 2: timed out */ - else { - elapsed_time = (trios_get_time_ms() - entry_time); - - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout > 0) && (elapsed_time >= timeout))) { - log_debug(debug_level, "MPI_Testany() timed out"); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - int timeout_remaining=timeout-elapsed_time; - if ((timeout < 0) || (timeout_remaining > MAX_SLEEP)) { - nnti_sleep(MAX_SLEEP); - } else { - if (timeout_remaining > 0) { - nnti_sleep(timeout_remaining); - } - } - - /* continue if the timeout has not expired */ - /* log_debug(debug_level, "timedout... continuing"); */ - - continue; - } - } - /* MPI_Testany() failure */ - else { - log_error(debug_level, "MPI_Testany() failed (request=%p): rc=%d", - mpi_wr->request_ptr, rc); - nnti_rc = NNTI_EIO; - break; - } - - if (is_any_wr_complete(wr_list, wr_count, which) == TRUE) { - break; - } - } - } - - - create_status(wr_list[*which], MPI_WORK_REQUEST(wr_list[*which]), nnti_rc, status); - - - if (nnti_rc==NNTI_OK) { - mpi_wr=MPI_WORK_REQUEST(wr_list[*which]); - assert(mpi_wr); - mpi_mem_hdl=MPI_MEM_HDL(mpi_wr->reg_buf); - assert(mpi_mem_hdl); - - switch (mpi_wr->last_op) { - case MPI_OP_NEW_REQUEST: - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.pop_front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - repost_recv_work_request(mpi_wr); - break; - case MPI_OP_PUT_TARGET: - case MPI_OP_GET_TARGET: - repost_rdma_target_work_request(mpi_wr); - break; - case MPI_OP_FETCH_ADD: - case MPI_OP_COMPARE_SWAP: - free(mpi_wr); - break; - default: - log_debug(nnti_debug_level, "status->offset=%llu, status->length=%llu", - (uint64_t)status->offset, (uint64_t)status->length); - uint32_t index=status->offset/status->length; - log_debug(nnti_debug_level, "status->offset=%llu, status->length=%llu, index=%llu", - (uint64_t)status->offset, (uint64_t)status->length, (uint64_t)index); - - mpi_mem_hdl=MPI_MEM_HDL(mpi_wr->reg_buf); - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - wr_queue_iter_t victim=find(mpi_mem_hdl->wr_queue.begin(), mpi_mem_hdl->wr_queue.end(), mpi_wr); - if (victim != mpi_mem_hdl->wr_queue.end()) { - log_debug(debug_level, "erasing mpi_wr(%p) from wr_queue", mpi_wr); - mpi_mem_hdl->wr_queue.erase(victim); - } - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - free(mpi_wr); - break; - } - } - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_mpi_wait", status); - } - -cleanup: - if (mpi_requests != NULL) free(mpi_requests); - if (request_to_wr_index != NULL) free(request_to_wr_index); - - log_debug(debug_level, "exit"); - - trios_stop_timer("NNTI_mpi_waitany", total_time); - - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on all buffers in wr_list to complete. - * - * Wait for remote_op on all buffers in wr_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in wr_list must be registered with the same transport. - * 2) You can't wait on the receive queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_mpi_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status) -{ - int rc=MPI_SUCCESS; - NNTI_result_t nnti_rc=NNTI_OK; - mpi_memory_handle *mpi_mem_hdl=NULL; - mpi_work_request *mpi_wr=NULL; - - uint32_t mpi_request_count=0; - MPI_Request *mpi_requests=NULL; - MPI_Status *events=NULL; - int done=FALSE; - - int ops_completed=0; - - long elapsed_time=0; -// long timeout_per_call; - - long entry_time=trios_get_time_ms(); - - log_level debug_level=nnti_debug_level; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - trios_start_timer(total_time); - - log_debug(debug_level, "enter"); - - assert(wr_list); - assert(wr_count > 0); - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_mpi_wait(wr_list[0], timeout, status[0]); - goto cleanup; - } - - for (uint32_t i=0;ireg_buf); - assert(mpi_mem_hdl); - mpi_wr=MPI_WORK_REQUEST(wr_list[i]); - if (mpi_wr==NULL) { - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_wr=mpi_mem_hdl->wr_queue.front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - wr_list[i]->transport_private=(uint64_t)mpi_wr; - } - assert(mpi_wr); - } - } - - if (is_all_wr_complete(wr_list, wr_count) == TRUE) { - log_debug(debug_level, "all work requests already complete (wr_list=%p)", wr_list); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "all work_requests NOT complete (wr_list=%p)", wr_list); - -// timeout_per_call = MIN_TIMEOUT; - - while (1) { - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - return NNTI_ECANCELED; - } - - ops_completed += check_atomic_operation(); - ops_completed += check_target_buffer_progress(); - - if (ops_completed > 0) { - ops_completed=0; - if (is_all_wr_complete(wr_list, wr_count) == TRUE) { - break; - } - } - - log_debug(debug_level, "waiting on wr_list(%p)", wr_list); - - /* - * The list of MPI_Requests is recreated each time through this loop. There is probably a better way. - */ - /* find the number of MPI_Requests in wr_list */ - mpi_request_count=0; - for (uint32_t i=0;irequest_count; - } - } - } - log_debug(debug_level, "wr_list (wr_count=%lu) contains %lu MPI_Requests", wr_count, mpi_request_count); - - mpi_requests=(MPI_Request *)realloc(mpi_requests, mpi_request_count * sizeof(MPI_Request)); - events =(MPI_Status *)realloc(events, mpi_request_count * sizeof(MPI_Status)); - assert(mpi_requests); - assert(events); - uint32_t request_index=0; - /* for each NNTI work request */ - for (uint32_t i=0;irequest_count;j++) { - mpi_requests[request_index++]=mpi_wr->request_ptr[j]; - } - } - } - } - - memset(events, 0, mpi_request_count*sizeof(MPI_Status)); - done=FALSE; - trios_start_timer(call_time); - nthread_lock(&nnti_mpi_lock); - rc = MPI_Testall(mpi_request_count, mpi_requests, &done, events); - nthread_unlock(&nnti_mpi_lock); - trios_stop_timer("NNTI_mpi_waitall - MPI_Testall", call_time); - log_debug(debug_level, "polling status is %d (rc=%d, done=%d)", rc, rc, done); - - if (done == TRUE) { - for (uint32_t i=0;i 0) && (elapsed_time >= timeout))) { - log_debug(debug_level, "MPI_Testany() timed out"); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - int timeout_remaining=timeout-elapsed_time; - if ((timeout < 0) || (timeout_remaining > MAX_SLEEP)) { - nnti_sleep(MAX_SLEEP); - } else { - if (timeout_remaining > 0) { - nnti_sleep(timeout_remaining); - } - } - - /* continue if the timeout has not expired */ - /* log_debug(debug_level, "timedout... continuing"); */ - - continue; - } - } - /* MPI_Testany() failure */ - else { - log_error(debug_level, "MPI_Testany() failed (request=%p): rc=%d", - mpi_wr->request_ptr, rc); - nnti_rc = NNTI_EIO; - break; - } - - if (is_all_wr_complete(wr_list, wr_count) == TRUE) { - break; - } - } - } - - - for (uint32_t i=0;ireg_buf); - assert(mpi_mem_hdl); - - create_status(wr_list[i], mpi_wr, nnti_rc, status[i]); - - if (nnti_rc == NNTI_OK) { - switch (mpi_wr->last_op) { - case MPI_OP_NEW_REQUEST: - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.pop_front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - repost_recv_work_request(mpi_wr); - break; - case MPI_OP_PUT_TARGET: - case MPI_OP_GET_TARGET: - repost_rdma_target_work_request(mpi_wr); - break; - case MPI_OP_FETCH_ADD: - case MPI_OP_COMPARE_SWAP: - free(mpi_wr); - break; - default: - log_debug(nnti_debug_level, "status->offset=%llu, status->length=%llu", - (uint64_t)status[i]->offset, (uint64_t)status[i]->length); - uint32_t index=status[i]->offset/status[i]->length; - log_debug(nnti_debug_level, "status->offset=%llu, status->length=%llu, index=%llu", - (uint64_t)status[i]->offset, (uint64_t)status[i]->length, (uint64_t)index); - - mpi_mem_hdl=MPI_MEM_HDL(mpi_wr->reg_buf); - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - wr_queue_iter_t victim=find(mpi_mem_hdl->wr_queue.begin(), mpi_mem_hdl->wr_queue.end(), mpi_wr); - if (victim != mpi_mem_hdl->wr_queue.end()) { - log_debug(debug_level, "erasing mpi_wr(%p) from wr_queue", mpi_wr); - mpi_mem_hdl->wr_queue.erase(victim); - } - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - free(mpi_wr); - break; - } - } - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status[i]", - "end of NNTI_mpi_wait", status[i]); - } - } - -cleanup: - if (mpi_requests != NULL) free(mpi_requests); - if (events != NULL) free(events); - - log_debug(debug_level, "exit"); - - trios_stop_timer("NNTI_mpi_waitall", total_time); - - return(nnti_rc); -} - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t NNTI_mpi_fini ( - const NNTI_transport_t *trans_hdl) -{ - nthread_counter_fini(&transport_global_data.mbits); - - nthread_lock_fini(&nnti_mpi_lock); - nthread_lock_fini(&nnti_buf_bufhash_lock); - nthread_lock_fini(&nnti_wr_wrhash_lock); - nthread_lock_fini(&nnti_target_buffer_queue_lock); - - if (transport_global_data.init_called_mpi_init) { - MPI_Finalize(); - } - - return(NNTI_OK); -} - - - -static NNTI_result_t setup_atomics(void) -{ - NNTI_result_t rc=NNTI_OK; /* return code */ - - trios_declare_timer(callTime); - - uint32_t atomics_bytes; - - log_debug(nnti_debug_level, "enter"); - - atomics_bytes=config.min_atomics_vars * sizeof(mpi_atomic_t); - trios_start_timer(callTime); - transport_global_data.atomics=(mpi_atomic_t*)malloc(atomics_bytes); - if (transport_global_data.atomics == NULL) { - rc=NNTI_ENOMEM; - goto cleanup; - } - memset(transport_global_data.atomics, 0, atomics_bytes); - trios_stop_timer("malloc and memset", callTime); - - trios_start_timer(callTime); - for (uint32_t i=0;iwr_queue_lock); - mpi_wr=mpi_mem_hdl->wr_queue.front(); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - log_debug(debug_level, "checking reg_buf(%p) mpi_wr(%p)", reg_buf, mpi_wr); - if (mpi_wr==NULL) { - log_debug(debug_level, "there are no work requests posted. continuing."); - continue; - } - if (is_wr_complete(mpi_wr) == TRUE) { - log_debug(debug_level, "this work request is complete. continuing."); - continue; - } - - log_debug(debug_level, "testing mpi_wr->reg_buf(%p) mpi_wr->request_ptr(%p)", mpi_wr->reg_buf , mpi_wr->request_ptr); - - memset(&event, 0, sizeof(MPI_Status)); - done=FALSE; - trios_start_timer(call_time); - nthread_lock(&nnti_mpi_lock); - rc = MPI_Testany(mpi_wr->request_count, mpi_wr->request_ptr, &which_req, &done, &event); - nthread_unlock(&nnti_mpi_lock); - trios_stop_timer("check_target_buffer_progress - MPI_Test", call_time); - if ((rc==MPI_SUCCESS) && (which_req==MPI_UNDEFINED) && (done==TRUE)) { - log_debug(debug_level, "MPI_Testany() says there a no active requests (rc=%d, which_req=%d, done=%d)", rc, which_req, done); - } - log_debug(debug_level, "polling status is %d, which_req=%d, done=%d", rc, which_req, done); - - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\tsource = %d", event.MPI_SOURCE); - log_debug(debug_level, "\ttag = %d", event.MPI_TAG); - log_debug(debug_level, "\terror = %d", event.MPI_ERROR); - log_debug(debug_level, "}"); - - if (rc == MPI_SUCCESS) { - /* case 1: success */ - if (done == 0) { - continue; - } - } - /* MPI_Test failure */ - else { - log_error(debug_level, "MPI_Test() failed (request=%p): rc=%d", - mpi_wr->request_ptr, rc); - break; - } - - process_event(mpi_wr, &event); - - if (((mpi_wr->last_op == MPI_OP_GET_TARGET) && (mpi_wr->op_state == RDMA_READ_COMPLETE)) || - ((mpi_wr->last_op == MPI_OP_PUT_TARGET) && (mpi_wr->op_state == RDMA_WRITE_COMPLETE))) { - - ops_completed++; - - // the op is complete - if (!(reg_buf->ops & NNTI_BOP_WITH_EVENTS)) { - // app doesn't want events, so we can recycle the work request - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - wr_queue_iter_t victim=find(mpi_mem_hdl->wr_queue.begin(), mpi_mem_hdl->wr_queue.end(), mpi_wr); - if (victim != mpi_mem_hdl->wr_queue.end()) { - log_debug(debug_level, "erasing mpi_wr(%p) from wr_queue", mpi_wr); - mpi_mem_hdl->wr_queue.erase(victim); - } - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - repost_rdma_target_work_request(mpi_wr); - } - } - } - nthread_unlock(&nnti_target_buffer_queue_lock); - - trios_stop_timer("check_target_buffer_progress", total_time); - - log_debug(debug_level, "exit"); - - return(ops_completed); -} - -static int check_atomic_operation(void) -{ - int ops_completed=0; - - int rc=MPI_SUCCESS; - - MPI_Status event; - int done=FALSE; - - int atomics_index =-1; - mpi_atomic_t *atomic=NULL; - - log_level debug_level=nnti_debug_level; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - trios_start_timer(total_time); - - log_debug(debug_level, "enter"); - - memset(&event, 0, sizeof(MPI_Status)); - done=FALSE; - trios_start_timer(call_time); - nthread_lock(&nnti_mpi_lock); - rc = MPI_Test(&transport_global_data.atomics_recv_request, &done, &event); - nthread_unlock(&nnti_mpi_lock); - trios_stop_timer("check_atomic_operation - MPI_Test", call_time); - log_debug(debug_level, "polling status is %d", rc); - - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\tsource = %d", event.MPI_SOURCE); - log_debug(debug_level, "\ttag = %d", event.MPI_TAG); - log_debug(debug_level, "\terror = %d", event.MPI_ERROR); - log_debug(debug_level, "}"); - log_debug(debug_level, "done = %d", done); - - if (rc == MPI_SUCCESS) { - /* case 1: success */ - if (done == 0) { - goto cleanup; - } - } - /* MPI_Test failure */ - else { - log_error(debug_level, "MPI_Test(atomics_recv_request) failed: rc=%d", rc); - goto cleanup; - } - - atomics_index=transport_global_data.atomics_request_msg.index; - atomic =&transport_global_data.atomics[atomics_index]; - - nthread_lock(&atomic->lock); - - switch (transport_global_data.atomics_request_msg.op) { - case MPI_ATOMIC_FETCH_ADD: - transport_global_data.atomics_result_msg.result = atomic->value; - atomic->value += transport_global_data.atomics_request_msg.compare_add; - break; - case MPI_ATOMIC_CMP_AND_SWP: - transport_global_data.atomics_result_msg.result=atomic->value; - if (atomic->value == transport_global_data.atomics_request_msg.compare_add) { - atomic->value = transport_global_data.atomics_request_msg.swap; - } - break; - default: - log_error(debug_level, "unknown atomic op: rc=%d", transport_global_data.atomics_request_msg.op); - break; - } - - nthread_lock(&nnti_mpi_lock); - rc=MPI_Send( - (char*)&transport_global_data.atomics_result_msg, - sizeof(transport_global_data.atomics_result_msg), - MPI_BYTE, - event.MPI_SOURCE, - NNTI_MPI_ATOMICS_RESULT_TAG, - MPI_COMM_WORLD); - nthread_unlock(&nnti_mpi_lock); - if (rc != MPI_SUCCESS) { - log_error(nnti_debug_level, "failed to send with Isend"); - goto cleanup; - } - - nthread_unlock(&atomic->lock); - - ops_completed++; - - post_atomics_recv_request(); - -cleanup: - trios_stop_timer("check_atomic_operation", total_time); - - log_debug(debug_level, "exit"); - - return(ops_completed); -} - - -static int process_event( - mpi_work_request *mpi_wr, - const MPI_Status *event) -{ - int rc=NNTI_OK; - NNTI_buffer_t *reg_buf =NULL; - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_level debug_level = nnti_debug_level; - - assert(mpi_wr); - - mpi_wr->last_event=*event; - - if ((mpi_wr->nnti_wr) && (mpi_wr->nnti_wr->ops == NNTI_BOP_ATOMICS)) { - if (mpi_wr->op_state == BUFFER_INIT) { - log_debug(debug_level, "got NNTI_BOP_ATOMICS send completion - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = SEND_COMPLETE; - mpi_wr->active_requests &= ~ATOMICS_SEND_REQUEST_ACTIVE; - - mpi_wr->request_ptr =&mpi_wr->request[ATOMICS_RECV_INDEX]; - mpi_wr->request_count=1; - - } else if (mpi_wr->op_state == SEND_COMPLETE) { - log_debug(debug_level, "got NNTI_BOP_ATOMICS recv completion - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RECV_COMPLETE; - mpi_wr->active_requests &= ~ATOMICS_RECV_REQUEST_ACTIVE; - } - - nthread_lock(&transport_global_data.atomics[mpi_wr->atomics_result_index].lock); - transport_global_data.atomics[mpi_wr->atomics_result_index].value = transport_global_data.atomics_result_msg.result; - nthread_unlock(&transport_global_data.atomics[mpi_wr->atomics_result_index].lock); - - mpi_wr->nnti_wr->result=NNTI_OK; - return NNTI_OK; - } - - reg_buf=mpi_wr->reg_buf; - assert(reg_buf); - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - if ((mpi_wr->op_state == BUFFER_INIT) && - (event->MPI_TAG == mpi_mem_hdl->cmd_tag)) { - /* this is an RDMA target work request. the command - * message tells us the operation being performed by - * the remote initiator. */ - mpi_wr->last_op = mpi_wr->cmd_msg.op; - } - - log_debug(debug_level, "mpi_wr=%p; mpi_wr->last_op=%d", mpi_wr, mpi_wr->last_op); - if (mpi_wr->last_op == MPI_OP_SEND_REQUEST) { - if (mpi_wr->op_state == BUFFER_INIT) { - log_debug(debug_level, "got SEND_REQUEST completion - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = SEND_COMPLETE; - mpi_wr->active_requests &= ~SEND_REQUEST_ACTIVE; - } - } else if (mpi_wr->last_op == MPI_OP_SEND_BUFFER) { - if (mpi_wr->op_state == BUFFER_INIT) { - log_debug(debug_level, "got SEND_BUFFER completion - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = SEND_COMPLETE; - mpi_wr->active_requests &= ~SEND_REQUEST_ACTIVE; - } - } else if (mpi_wr->last_op == MPI_OP_NEW_REQUEST) { - if (mpi_wr->op_state == BUFFER_INIT) { - log_debug(debug_level, "got NEW REQUEST completion - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RECV_COMPLETE; - mpi_wr->dst_offset = mpi_wr->request_index*mpi_wr->length; - } - } else if (mpi_wr->last_op == MPI_OP_PUT_INITIATOR) { - if (mpi_wr->op_state == RDMA_WRITE_INIT) { - log_debug(debug_level, "got put_src RTS completion (initiator) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_RTS_COMPLETE; - mpi_wr->active_requests &= ~RDMA_CMD_REQUEST_ACTIVE; - - mpi_wr->request_ptr =&mpi_wr->request[PUT_SEND_INDEX]; - mpi_wr->request_count=1; - - } else if (mpi_wr->op_state == RDMA_RTS_COMPLETE) { - log_debug(debug_level, "got put_src WRITE completion (initiator) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_WRITE_COMPLETE; - mpi_wr->active_requests &= ~PUT_SEND_REQUEST_ACTIVE; - } - } else if (mpi_wr->last_op == MPI_OP_PUT_TARGET) { - if (mpi_wr->op_state == BUFFER_INIT) { - log_debug(debug_level, "got put_dst RTS completion (target) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_RTS_COMPLETE; - mpi_wr->active_requests &= ~RDMA_CMD_REQUEST_ACTIVE; - - log_debug(debug_level, "receiving data from PUT initiator - rank(%d) tag(%d) dst_offset(%llu) dst_length(%llu)", - event->MPI_SOURCE, mpi_wr->cmd_msg.tag, - mpi_wr->cmd_msg.offset, mpi_wr->cmd_msg.length); - nthread_lock(&nnti_mpi_lock); - MPI_Irecv( - (char*)reg_buf->payload+mpi_wr->cmd_msg.offset, - mpi_wr->cmd_msg.length, - MPI_BYTE, - event->MPI_SOURCE, - mpi_mem_hdl->put_data_tag, - MPI_COMM_WORLD, - &mpi_wr->request[PUT_RECV_INDEX]); - nthread_unlock(&nnti_mpi_lock); - mpi_wr->request_ptr =&mpi_wr->request[PUT_RECV_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= PUT_RECV_REQUEST_ACTIVE; - - mpi_wr->dst_offset=mpi_wr->cmd_msg.offset; - mpi_wr->length =mpi_wr->cmd_msg.length; - - } else if (mpi_wr->op_state == RDMA_RTS_COMPLETE) { - log_debug(debug_level, "got put_dst WRITE completion (target) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_WRITE_COMPLETE; - mpi_wr->active_requests &= ~PUT_RECV_REQUEST_ACTIVE; - } - } else if (mpi_wr->last_op == MPI_OP_GET_INITIATOR) { - if (mpi_wr->op_state == RDMA_READ_INIT) { - log_debug(debug_level, "got get_dst RTR completion (initiator) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_RTR_COMPLETE; - mpi_wr->active_requests &= ~RDMA_CMD_REQUEST_ACTIVE; - - mpi_wr->request_ptr =&mpi_wr->request[GET_RECV_INDEX]; - mpi_wr->request_count=1; - - } else if (mpi_wr->op_state == RDMA_RTR_COMPLETE) { - log_debug(debug_level, "got get_dst READ completion (initiator) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_READ_COMPLETE; - mpi_wr->active_requests &= ~GET_RECV_REQUEST_ACTIVE; - } - } else if (mpi_wr->last_op == MPI_OP_GET_TARGET) { - if (mpi_wr->op_state == BUFFER_INIT) { - log_debug(debug_level, "got get_src RTR completion (target) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_RTR_COMPLETE; - mpi_wr->active_requests &= ~RDMA_CMD_REQUEST_ACTIVE; - - log_debug(debug_level, "sending data to GET initiator - rank(%d) tag(%d) src_offset(%llu) src_length(%llu)", - event->MPI_SOURCE, mpi_wr->cmd_msg.tag, - mpi_wr->cmd_msg.offset, mpi_wr->cmd_msg.length); - nthread_lock(&nnti_mpi_lock); - MPI_Issend( - (char*)reg_buf->payload+mpi_wr->cmd_msg.offset, - mpi_wr->cmd_msg.length, - MPI_BYTE, - event->MPI_SOURCE, - mpi_wr->cmd_msg.tag, - MPI_COMM_WORLD, - &mpi_wr->request[GET_SEND_INDEX]); - nthread_unlock(&nnti_mpi_lock); - mpi_wr->request_ptr =&mpi_wr->request[GET_SEND_INDEX]; - mpi_wr->request_count=1; - mpi_wr->active_requests |= GET_SEND_REQUEST_ACTIVE; - - mpi_wr->src_offset=mpi_wr->cmd_msg.offset; - mpi_wr->length =mpi_wr->cmd_msg.length; - } else if (mpi_wr->op_state == RDMA_RTR_COMPLETE) { - log_debug(debug_level, "got get_src READ completion (target) - event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - - mpi_wr->op_state = RDMA_READ_COMPLETE; - mpi_wr->active_requests &= ~GET_SEND_REQUEST_ACTIVE; - } - } else { - log_error(debug_level, "unrecognized event arrived from %d - tag %4d", - event->MPI_SOURCE, event->MPI_TAG); - } - - return (rc); -} - -static NNTI_result_t post_recv_queue_work_request( - NNTI_buffer_t *reg_buf, - int64_t tag, - uint64_t length, - uint64_t count, - MPI_Request *request_list) -{ - uint64_t i=0; - mpi_work_request *mpi_wr=NULL; - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - mpi_wr->reg_buf =reg_buf; - mpi_wr->dst_offset =0; - mpi_wr->length =length; - mpi_wr->tag =tag; - mpi_wr->request_ptr =request_list; - mpi_wr->request_count=count; - mpi_wr->op_state =BUFFER_INIT; - - for (i=0;ilast_op=MPI_OP_NEW_REQUEST; - - log_debug(nnti_debug_level, "posting irecv (reg_buf=%p ; mpi_wr=%p ; request_ptr=%p, tag=%lld)", reg_buf, mpi_wr, &mpi_wr->request_ptr[i], mpi_wr->tag); - nthread_lock(&nnti_mpi_lock); - MPI_Irecv( - (char*)reg_buf->payload + offset, - length, - MPI_BYTE, - MPI_ANY_SOURCE, - mpi_wr->tag, - MPI_COMM_WORLD, - &(request_list[i])); - nthread_unlock(&nnti_mpi_lock); - } - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static NNTI_result_t post_rdma_target_work_request( - NNTI_buffer_t *reg_buf) -{ - mpi_work_request *mpi_wr=NULL; - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - mpi_wr=(mpi_work_request *)calloc(1, sizeof(mpi_work_request)); - assert(mpi_wr); - - mpi_wr->reg_buf =reg_buf; - mpi_wr->op_state=BUFFER_INIT; - - mpi_wr->request_count=0; - if ((reg_buf->ops & NNTI_BOP_REMOTE_READ) || - (reg_buf->ops & NNTI_BOP_REMOTE_WRITE)) { - mpi_wr->request_ptr = &mpi_wr->request[RDMA_CMD_INDEX]; - mpi_wr->request_count=1; - - nthread_lock(&nnti_mpi_lock); - MPI_Irecv( - &mpi_wr->cmd_msg, - sizeof(mpi_wr->cmd_msg), - MPI_BYTE, - MPI_ANY_SOURCE, - mpi_mem_hdl->cmd_tag, - MPI_COMM_WORLD, - &mpi_wr->request[RDMA_CMD_INDEX]); - nthread_unlock(&nnti_mpi_lock); - - mpi_wr->active_requests |= RDMA_CMD_REQUEST_ACTIVE; - } - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "exit (reg_buf=%p ; mpi_wr->request_ptr=%p)", reg_buf, mpi_wr->request_ptr); - - return(NNTI_OK); -} - -static NNTI_result_t post_atomics_recv_request(void) -{ - log_debug(nnti_debug_level, "enter"); - - nthread_lock(&nnti_mpi_lock); - MPI_Irecv( - &transport_global_data.atomics_request_msg, - sizeof(transport_global_data.atomics_request_msg), - MPI_BYTE, - MPI_ANY_SOURCE, - NNTI_MPI_ATOMICS_REQUEST_TAG, - MPI_COMM_WORLD, - &transport_global_data.atomics_recv_request); - nthread_unlock(&nnti_mpi_lock); - - log_debug(nnti_debug_level, "exit"); - - return(NNTI_OK); -} - - -static NNTI_result_t repost_recv_work_request( - mpi_work_request *mpi_wr) -{ - NNTI_buffer_t *reg_buf =mpi_wr->reg_buf; - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - assert(reg_buf); - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - mpi_wr->op_state=BUFFER_INIT; - mpi_wr->active_requests |= RECV_REQUEST_ACTIVE; - - log_debug(nnti_debug_level, "posting irecv (reg_buf=%p ; mpi_wr=%p ; request_ptr=%p, tag=%lld)", reg_buf, mpi_wr, mpi_wr->request_ptr, mpi_wr->tag); - nthread_lock(&nnti_mpi_lock); - MPI_Irecv( - (char*)reg_buf->payload + (mpi_wr->request_index * mpi_wr->length), - mpi_wr->length, - MPI_BYTE, - MPI_ANY_SOURCE, - mpi_wr->tag, - MPI_COMM_WORLD, - &mpi_wr->request_ptr[mpi_wr->request_index]); - nthread_unlock(&nnti_mpi_lock); - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - -static NNTI_result_t repost_rdma_target_work_request( - mpi_work_request *mpi_wr) -{ - NNTI_buffer_t *reg_buf =mpi_wr->reg_buf; - mpi_memory_handle *mpi_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - assert(reg_buf); - mpi_mem_hdl=MPI_MEM_HDL(reg_buf); - assert(mpi_mem_hdl); - - mpi_wr->op_state=BUFFER_INIT; - - mpi_wr->request_count=0; - if ((reg_buf->ops & NNTI_BOP_REMOTE_READ) || - (reg_buf->ops & NNTI_BOP_REMOTE_WRITE)) { - mpi_wr->request_ptr = &mpi_wr->request[RDMA_CMD_INDEX]; - mpi_wr->request_count=1; - - nthread_lock(&nnti_mpi_lock); - MPI_Irecv( - &mpi_wr->cmd_msg, - sizeof(mpi_wr->cmd_msg), - MPI_BYTE, - MPI_ANY_SOURCE, - mpi_mem_hdl->cmd_tag, - MPI_COMM_WORLD, - &mpi_wr->request[RDMA_CMD_INDEX]); - nthread_unlock(&nnti_mpi_lock); - - mpi_wr->active_requests |= RDMA_CMD_REQUEST_ACTIVE; - } - - nthread_lock(&mpi_mem_hdl->wr_queue_lock); - mpi_mem_hdl->wr_queue.push_back(mpi_wr); - nthread_unlock(&mpi_mem_hdl->wr_queue_lock); - - log_debug(nnti_debug_level, "exit (reg_buf=%p ; mpi_wr->request_ptr=%p)", reg_buf, mpi_wr->request_ptr); - - return(NNTI_OK); -} - - -static int is_wr_complete( - mpi_work_request *mpi_wr) -{ - int8_t rc=FALSE; - - assert(mpi_wr); - - switch (mpi_wr->last_op) { - case MPI_OP_PUT_INITIATOR: - if (mpi_wr->op_state == RDMA_WRITE_COMPLETE) { - rc=TRUE; - } - break; - case MPI_OP_GET_INITIATOR: - if (mpi_wr->op_state == RDMA_READ_COMPLETE) { - rc=TRUE; - } - break; - case MPI_OP_PUT_TARGET: - if (mpi_wr->op_state == RDMA_WRITE_COMPLETE) { - rc=TRUE; - } - break; - case MPI_OP_GET_TARGET: - if (mpi_wr->op_state == RDMA_READ_COMPLETE) { - rc=TRUE; - } - break; - case MPI_OP_SEND_REQUEST: - case MPI_OP_SEND_BUFFER: - if (mpi_wr->op_state == SEND_COMPLETE) { - rc=TRUE; - } - break; - case MPI_OP_NEW_REQUEST: - if (mpi_wr->op_state == RECV_COMPLETE) { - rc=TRUE; - } - break; - case MPI_OP_FETCH_ADD: - case MPI_OP_COMPARE_SWAP: - if (mpi_wr->op_state == RECV_COMPLETE) { - rc=TRUE; - } - break; - default: - break; - } - - log_debug(nnti_debug_level, "exit (rc=%d)", rc); - return(rc); -} - -static int8_t is_wr_complete( - NNTI_work_request_t *wr) -{ - mpi_work_request *mpi_wr=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - mpi_wr=MPI_WORK_REQUEST(wr); - assert(mpi_wr); - - return(is_wr_complete(mpi_wr)); -} - -static int8_t is_any_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - uint32_t *which) -{ - int8_t rc=FALSE; - - log_debug(nnti_debug_level, "enter"); - - for (uint32_t i=0;iop =wr->ops; - status->result=(NNTI_result_t)nnti_rc; - if (nnti_rc==NNTI_OK) { - if (mpi_wr->reg_buf) { - status->start =mpi_wr->reg_buf->payload; - status->length=mpi_wr->length; - } - status->result=(NNTI_result_t)nnti_rc; - switch (mpi_wr->last_op) { - case MPI_OP_PUT_INITIATOR: - case MPI_OP_GET_TARGET: - case MPI_OP_SEND_REQUEST: - case MPI_OP_SEND_BUFFER: - status->offset=mpi_wr->src_offset; - create_peer(&status->src, transport_global_data.rank); // allocates url - create_peer(&status->dest, mpi_wr->last_event.MPI_SOURCE); // allocates url - break; - case MPI_OP_GET_INITIATOR: - case MPI_OP_PUT_TARGET: - case MPI_OP_NEW_REQUEST: - status->offset=mpi_wr->dst_offset; - create_peer(&status->src, mpi_wr->last_event.MPI_SOURCE); // allocates url - create_peer(&status->dest, transport_global_data.rank); // allocates url - break; - } - } - - log_debug(nnti_debug_level, "exit"); -} - -static void create_peer(NNTI_peer_t *peer, int rank) -{ - log_debug(nnti_debug_level, "enter"); - - sprintf(peer->url, "mpi://%u/", rank); - - peer->peer.transport_id = NNTI_TRANSPORT_MPI; - peer->peer.NNTI_remote_process_t_u.mpi.rank = rank; - - log_debug(nnti_debug_level, "exit"); -} - -static void config_init(nnti_mpi_config *c) -{ - c->min_atomics_vars = 512; -} - -static void config_get_from_env(nnti_mpi_config *c) -{ - char *env_str=NULL; - - // defaults - c->min_atomics_vars = 512; - - if ((env_str=getenv("TRIOS_NNTI_MIN_ATOMIC_VARS")) != NULL) { - errno=0; - uint32_t min_vars=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(nnti_debug_level, "setting c->min_atomics_vars to %lu", min_vars); - c->min_atomics_vars=min_vars; - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MIN_ATOMIC_VARS value conversion failed (%s). using c->min_atomics_vars default.", strerror(errno)); - } - } else { - log_debug(nnti_debug_level, "TRIOS_NNTI_MIN_ATOMIC_VARS is undefined. using c->min_atomics_vars default"); - } -} diff --git a/packages/trios/libraries/nessie/nnti/nnti_mpi.h b/packages/trios/libraries/nessie/nnti/nnti_mpi.h deleted file mode 100644 index 284e10ed9170..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_mpi.h +++ /dev/null @@ -1,232 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_mpi.h - * - * Created on: Apr 4, 2012 - * Author: thkorde - */ - -#ifndef NNTI_MPI_H_ -#define NNTI_MPI_H_ - -#include "Trios_config.h" - -#include "Trios_nnti.h" -#include "nnti_internal.h" - - -#if defined(HAVE_TRIOS_MPI) -#include -#endif - - -#define MPI_SET_MATCH_ANY(p) \ -(p)->peer.transport_id = NNTI_TRANSPORT_MPI; \ -(p)->peer.NNTI_remote_process_t_u.mpi.rank = MPI_ANY_SOURCE; - - -#ifdef __cplusplus -extern "C" { -#endif - -NNTI_result_t NNTI_mpi_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_mpi_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -NNTI_result_t NNTI_mpi_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_mpi_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_mpi_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_mpi_free ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_mpi_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_mpi_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_mpi_unregister_memory ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_mpi_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context); - -NNTI_result_t NNTI_mpi_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value); - -NNTI_result_t NNTI_mpi_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_clear_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_destroy_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_cancel ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_mpi_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -NNTI_result_t NNTI_mpi_interrupt ( - const NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_mpi_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status); - -NNTI_result_t NNTI_mpi_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -NNTI_result_t NNTI_mpi_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status); - -NNTI_result_t NNTI_mpi_fini ( - const NNTI_transport_t *trans_hdl); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_MPI_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/nnti_pami.cpp b/packages/trios/libraries/nessie/nnti/nnti_pami.cpp deleted file mode 100644 index 118cbe73979f..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_pami.cpp +++ /dev/null @@ -1,2998 +0,0 @@ -/* - * nnti_pami.c - * - * Created on: April 19, 2012 - */ - -#include "Trios_config.h" -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include "nnti_pami.h" -#include "nnti_utils.h" -#include "nnti_internal.h" - -#define TRACE_ERR(x) /* fprintf x */ - -/* GLOBAL VAR */ - - -#define BUFSIZE 2048 -#define ITERATIONS 1000 -#define MAX_CONNECTION 1024 -size_t my_rank; -int is_client = 0; -volatile unsigned _recv_active; -volatile unsigned send_active; -volatile unsigned _recv_iteration; -char _recv_buffer[BUFSIZE] __attribute__ ((__aligned__ (16))); -volatile static int insert_conn_index; -int conn_list[1024][2]; /*Populate this with rank and need Attr during dispatch_recv immediate */ -int cur_conn_idx =0; -int conn_ack =0; -pami_context_t * contexts; -size_t dispatch_id = 37; -pthread_t Progress_thread; -/*** End of GLOBAL VAR ***/ - - -/** - * These states are used to signal events between the completion handler - * and the main client or server thread. - * - * Once CONNECTED, they cycle through RDMA_READ_ADV, RDMA_WRITE_ADV, - * and RDMA_WRITE_COMPLETE for each ping. - */ -typedef enum -{ - IDLE = 1, - CONNECT_REQUEST, - CONNECTED, - DISCONNECTED, - ERROR -} bgqpami_connection_state; - -typedef enum -{ - SERVER_CONNECTION, - CLIENT_CONNECTION -} bgqpami_connection_type; - -typedef enum -{ - REQUEST_BUFFER, - RESULT_BUFFER, - RECEIVE_BUFFER, - SEND_BUFFER, - GET_SRC_BUFFER, - GET_DST_BUFFER, - PUT_SRC_BUFFER, - PUT_DST_BUFFER, - RDMA_TARGET_BUFFER, - UNKNOWN_BUFFER -} bgqpami_buffer_type; - - -#define PAMI_OP_PUT_INITIATOR 1 -#define PAMI_OP_GET_INITIATOR 2 -#define PAMI_OP_PUT_TARGET 3 -#define PAMI_OP_GET_TARGET 4 -#define PAMI_OP_SEND_REQ 5 -#define PAMI_OP_NEW_REQUEST 6 -#define PAMI_OP_RESULT 7 -#define PAMI_OP_RECEIVE 8 -#define PAMI_OP_REGISTER_RDMA 9 - -typedef enum -{ - SEND_COMPLETE, - RECV_COMPLETE, - RDMA_WRITE_INIT, - RDMA_TARGET_INIT, - RDMA_READ_INIT, - RDMA_COMPLETE -} bgqpami_op_state_t; - - -typedef struct -{ - - size_t req_rank; - pami_memregion_t response_hdl; -} nnti_bgqpami_fetch_index_req; - -typedef struct -{ - uint32_t req_received; - int local_send_complete; - int local_get_complete; - int remote_put_complete; - int op_complete; - uint32_t req_rank; - uint64_t byte_len; - uint64_t byte_offset; - uint64_t src_offset; - uint64_t dest_offset; - uint64_t dest_addr; - uint16_t op; -} nnti_bgqpami_work_completion; - -/** - * attrs to send to the client - */ -typedef struct -{ - uint64_t req_index_addr; /* address of the request index var on the server */ - pami_memregion_t req_index_mem_hdl; - - uint64_t req_buffer_addr; /* address of the request buffer */ - uint64_t req_size; /* the maximum size of a request */ - uint64_t req_count; /* the number of requests that will fit in the queue */ - pami_memregion_t req_mem_hdl; - uint64_t wc_buffer_addr; /* address of the work completion array */ - pami_memregion_t wc_mem_hdl; - uint64_t unblock_buffer_addr; /* address of the fetch_req_array */ - pami_memregion_t unblock_mem_hdl; -} nnti_bgqpami_server_queue_attrs; - -/** - * attrs to send to the server - */ -typedef struct -{ - uint64_t req_index; /* this buffer contains the current index of the request queue */ - uint64_t req_index_addr; /* address of the index buffer */ - pami_memregion_t req_index_mem_hdl; /* mem hdl to send to the server */ - pami_memregion_t fetch_index_hdl; -} nnti_bgqpami_client_queue_attrs; - -typedef union -{ - nnti_bgqpami_client_queue_attrs client; - nnti_bgqpami_server_queue_attrs server; -} nnti_bgqpami_queue_remote_attrs; - -typedef struct -{ - uint32_t peer_rank; - nnti_bgqpami_client_queue_attrs queue_local_attrs; - nnti_bgqpami_queue_remote_attrs queue_remote_attrs; - bgqpami_connection_state state; - bgqpami_connection_type connection_type; -} bgqpami_connection; - - -typedef struct -{ - uint8_t is_initiator; - - const NNTI_buffer_t *reg_buf; - nnti_bgqpami_work_completion wc; - uint64_t wc_dest_addr; - pami_memregion_t wc_dest_mem_hdl; - pami_memregion_t wc_mem_hdl; - uint8_t last_op; - bgqpami_op_state_t op_state; - uint64_t is_last_op_complete; - uint32_t peer_rank; -} pami_work_request; - - -typedef - std::deque < -pami_work_request * > - wr_queue_t; -typedef - std::deque < -pami_work_request * >::iterator - wr_queue_iter_t; - -typedef struct -{ - bgqpami_buffer_type - type; - pami_memregion_t - mem_hdl; /* actual data memory handle */ - wr_queue_t - wr_queue; - nthread_lock_t - wr_queue_lock; - uint32_t - ref_count; -} bgqpami_memory_handle; - -typedef struct -{ - NNTI_buffer_t * - reg_buf; - - uint64_t - last_index_before_reset; - uint64_t - req_index; /* index of the next available slot in the request queue */ - uint64_t - req_index_addr; /* address of the request index var on the server */ - pami_memregion_t - req_index_mem_hdl; - - char * - req_buffer; /* pointer to the head of the request buffer */ - uint64_t - req_size; /* the maximum size of a request */ - uint64_t - req_count; /* the number of requests that will fit in the queue */ - uint64_t - req_buffer_size; /* the size of the request buffer in bytes (req_size*req_count) */ - pami_memregion_t - req_mem_hdl; - - nnti_bgqpami_work_completion * - wc_buffer; /* pointer to work completion array */ - uint64_t - wc_buffer_size; /* the size of the work completion buffer in bytes */ - pami_memregion_t - wc_mem_hdl; - nnti_bgqpami_fetch_index_req * - unblock_buffer; /* pointer to individual fetch index array */ - pami_memregion_t - unblock_mem_hdl; - uint64_t - req_processed_reset_limit; - uint64_t - req_processed; - uint64_t - total_req_processed; -} bgqpami_request_queue_handle; - -typedef struct -{ - uint32_t - myrank; - uint32_t - remote_rank; - int - mypid; - bgqpami_request_queue_handle - req_queue; -} bgqpami_transport_global; - -static nthread_lock_t - nnti_bgqpami_lock; -static nthread_lock_t - nnti_index_lock; -static int -start_connection_listener_thread (void); -static NNTI_result_t -register_memory (bgqpami_memory_handle * hdl, - void *buf, uint64_t len, const NNTI_buf_ops_t remote_op); -static int -unregister_memory (bgqpami_memory_handle * hdl); -static NNTI_result_t -process_event (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - pami_work_request * wr, const int timeout); -/* NEW SHYAMALI */ - -static NNTI_result_t -repost_recv_work_request (NNTI_buffer_t * reg_buf, pami_work_request * wr); - -static pami_work_request * -first_incomplete_wr (bgqpami_memory_handle * _hdl); -static int8_t -is_buf_op_complete (const NNTI_buffer_t * reg_buf); -static int8_t -is_wr_complete (pami_work_request * wr); -static int8_t -is_any_buf_op_complete (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, uint32_t * which); -static int8_t -is_all_buf_ops_complete (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count); -static void -create_status (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - int nnti_rc, NNTI_status_t * status); - -/* NEW */ -static void -create_peer (NNTI_peer_t * peer, int rank); -static NNTI_result_t -recv_full (int rank, void *buf, size_t num); -static NNTI_result_t -init_connection (bgqpami_connection ** conn, int rank, const int is_server); -static void -close_connection (bgqpami_connection * c); - -static NNTI_result_t -insert_conn_rank (const uint32_t rank, bgqpami_connection * conn); -static bgqpami_connection * -get_conn_rank (const uint32_t rank); -static bgqpami_connection * -del_conn_rank (const uint32_t rank); - -static int -server_req_queue_init (bgqpami_request_queue_handle * q, - char *buffer, uint64_t req_size, uint64_t req_count); -static int -server_req_queue_destroy (bgqpami_request_queue_handle * q); - -static int -client_req_queue_init (bgqpami_connection * c); -static int -client_req_queue_destroy (bgqpami_connection * c); - -static int -reset_req_index (bgqpami_request_queue_handle * req_queue_attrs); - -static int -fetch_server_req_buffer_offset (nnti_bgqpami_client_queue_attrs * - local_req_queue_attrs, - nnti_bgqpami_server_queue_attrs * - remote_req_queue_attrs, - uint64_t addend, uint64_t * offset, int rank); -static int -send_req (nnti_bgqpami_client_queue_attrs * local_req_queue_attrs, - nnti_bgqpami_server_queue_attrs * remote_req_queue_attrs, - uint64_t offset, const NNTI_buffer_t * reg_buf, - int rank, pami_work_request * wr); - -static int -send_req_wc (nnti_bgqpami_client_queue_attrs * local_req_queue_attrs, - nnti_bgqpami_server_queue_attrs * remote_req_queue_attrs, - const NNTI_buffer_t * reg_buf, uint64_t offset, pami_work_request * wr, int rank); - -static int -request_send (nnti_bgqpami_client_queue_attrs * client_q, - nnti_bgqpami_server_queue_attrs * server_q, - const NNTI_buffer_t * reg_buf, int rank); - -/* NEW */ -static NNTI_result_t -register_wc (pami_work_request * wr); -static NNTI_result_t -insert_buf_bufhash (NNTI_buffer_t * buf); -static NNTI_buffer_t * -get_buf_bufhash (const uint32_t bufhash); -static NNTI_buffer_t * -del_buf_bufhash (NNTI_buffer_t * buf); -static void -print_bufhash_map (void); - -static NNTI_result_t -insert_wr_wrhash (pami_work_request *); -static pami_work_request * -get_wr_wrhash (const uint32_t bufhash); -static pami_work_request * -del_wr_wrhash (pami_work_request *); -static void -print_wrhash_map (void); -static void -send_rdma_wc (pami_work_request * wr, - const NNTI_buffer_t * local_buf, - const NNTI_buffer_t * remote_buf); - - -static - NNTI_result_t -new_server_connection (bgqpami_connection * c, int peer_rank); -/* Thomas Wang's 64 bit to 32 bit Hash Function (http://www.concentric.net/~ttwang/tech/inthash.htm) */ -static uint32_t -hash6432shift (uint64_t key) -{ - key = (~key) + (key << 18); // key = (key << 18) - key - 1; - key = key ^ (key >> 31); - key = key * 21; // key = (key + (key << 2)) + (key << 4); - key = key ^ (key >> 11); - key = key + (key << 6); - key = key ^ (key >> 22); - return (uint32_t) key; -} - -/* - * We need a couple of maps to keep track of connections. Servers need to find - * connections by QP number when requests arrive. Clients need to find connections - * by peer address and port. Setup those maps here. - */ -static - std::map < int, -bgqpami_connection * >connections_by_rank; -typedef - std::map < int, -bgqpami_connection * >::iterator - conn_by_rank_iter_t; -typedef - std::pair < int, -bgqpami_connection * > - conn_by_rank_t; -static nthread_lock_t - nnti_conn_peer_lock; - -static int8_t -is_wr_queue_empty (const NNTI_buffer_t * buf); - -static - std::map < -uint32_t, NNTI_buffer_t * >buffers_by_bufhash; -typedef - std::map < - uint32_t, -NNTI_buffer_t * >::iterator - buf_by_bufhash_iter_t; -typedef - std::pair < - uint32_t, -NNTI_buffer_t * > - buf_by_bufhash_t; -static nthread_lock_t - nnti_buf_bufhash_lock; - -static - std::map < - uint32_t, -pami_work_request * > - wr_by_wrhash; -typedef - std::map < - uint32_t, -pami_work_request * >::iterator - wr_by_wrhash_iter_t; -typedef - std::pair < - uint32_t, -pami_work_request * > - wr_by_wrhash_t; -static nthread_lock_t - nnti_wr_wrhash_lock; - - - -/* NEW */ - -static bgqpami_transport_global - transport_global_data; -static const int - MIN_TIMEOUT = 80000; /* in milliseconds */ -static log_level - nnti_event_debug_level; -static nnti_bgqpami_fetch_index_req - client_fetch_req; - -/************************** Callback functions for PAMI_Send / Recv protocol ************/ -/* Low overhead timing function */ - -static void * Progress_function(void * dummy) -{ - pami_result_t result = PAMI_ERROR; - - while (1) - { - result = PAMI_Context_trylock_advancev(&(contexts[1]), 5, 1000); - usleep(100); - } - - return NULL; -} - - - -void * -safemalloc(size_t size) -{ - void * - ptr; - int - rc; - - rc = posix_memalign (&ptr, 128, size); - if (rc) - return NULL; - - return ptr; -} - - -void dispatch_done_cb(pami_context_t context, void * cookie, pami_result_t result) -{ - - fflush(stdout); - return; -} -static void cb_done (void *ctxt, void * clientdata, pami_result_t err) -{ - int * active = (int *) clientdata; - (*active)--; -} - -void dispatch_recv_cb(pami_context_t context, - void * cookie, - const void * header_addr, size_t header_size, - const void * pipe_addr, - size_t data_size, - pami_endpoint_t origin, - pami_recv_t * recv) -{ - pami_result_t result = PAMI_ERROR; - bgqpami_connection *conn; - - size_t task; - size_t ctxoff; - result = PAMI_Endpoint_query(origin, &task, &ctxoff); - - if ((data_size < 32 ) || (data_size > 80 )) - return ; - conn_ack--; - - if (pipe_addr!=NULL) - { - - nthread_lock (&nnti_bgqpami_lock); - conn = get_conn_rank (origin); - if (conn == NULL) { - if(is_client != 1){ - conn = (bgqpami_connection *) calloc (1, sizeof (bgqpami_connection)); - conn->peer_rank = origin; - memcpy((void *)&conn->queue_remote_attrs.client, pipe_addr, data_size); - insert_conn_rank (conn->peer_rank, conn); - - log_debug (nnti_debug_level, - "accepted new connection from 192.168.1.%d", conn->peer_rank); - conn_list[cur_conn_idx][0] = origin; - conn_list[cur_conn_idx][1] = 0; - cur_conn_idx++; - } - } - else if((is_client == 1) && (conn != NULL)) { - memcpy((void *)&conn->queue_remote_attrs.server, pipe_addr, data_size); - conn->peer_rank = origin; - - } - nthread_unlock (&nnti_bgqpami_lock); - } - else - { - fflush(stdout); - nthread_lock (&nnti_bgqpami_lock); - conn = get_conn_rank (origin); - if (conn == NULL) { - conn = (bgqpami_connection *) calloc (1, sizeof (bgqpami_connection)); - } - conn->peer_rank = origin; - insert_conn_rank (conn->peer_rank, conn); - - recv->cookie = (void *) conn; - recv->local_fn = dispatch_done_cb; - if(is_client == 1) - recv->addr = &conn->queue_remote_attrs.server; - recv->type = PAMI_TYPE_BYTE; - recv->offset = 0; - recv->data_fn = PAMI_DATA_COPY; - recv->data_cookie = NULL; - nthread_unlock (&nnti_bgqpami_lock); - } - - return; -} - -static void decrement(void *ctxt, void * clientdata, pami_result_t err) -{ - int * active = (int *) clientdata; - (*active)--; -} - - -void -send_once (char *buffer, size_t sndlen, size_t targetrank) -{ - pami_endpoint_t target_ep; - pami_result_t result = PAMI_Endpoint_create(client, (pami_task_t) targetrank, 1, &target_ep); - - // void *iov_base - Contains the address of a buffer. - // size_t iov_len - Contains the length of the buffer. - - int header = 37373; - - pami_send_t parameters; - parameters.send.header.iov_base = &header; - parameters.send.header.iov_len = sizeof(int); - parameters.send.data.iov_base = buffer; - parameters.send.data.iov_len = sndlen; - parameters.send.dispatch = dispatch_id; - parameters.send.dest = target_ep; - parameters.events.cookie = (void *)&_recv_active; - parameters.events.local_fn = cb_done; - parameters.events.remote_fn = NULL;//cb_done; - - uint64_t t0 = GetTimeBase(); - _recv_active = 1; - - result = PAMI_Send(contexts[0], ¶meters); - - while (_recv_active) - { - result = PAMI_Context_trylock_advancev(&(contexts[0]), 1, 1000); - } -} - - - -/* ==================== */ - - - - -/** - * @brief Initialize NNTI to use a specific transport. - * - */ -NNTI_result_t -NNTI_bgqpami_init (const NNTI_transport_id_t trans_id, - const char *my_url, NNTI_transport_t * trans_hdl) -{ - static int - initialized = 0; - - NNTI_result_t - rc = NNTI_OK; - - pami_result_t result = PAMI_ERROR; - size_t world_size, world_rank; - int i; - - /* initialize the second client */ - char clientname[20] = "NNTI"; - pami_client_t client; - result = PAMI_Client_create((char *)&clientname, &client, NULL, 0); - - /* query properties of the client */ - pami_configuration_t config[3]; - size_t num_contexts; - - config[0].name = PAMI_CLIENT_NUM_TASKS; - config[1].name = PAMI_CLIENT_TASK_ID; - config[2].name = PAMI_CLIENT_NUM_CONTEXTS; - result = PAMI_Client_query(client, config, 3); - world_size = config[0].value.intval; - my_rank = config[1].value.intval; - num_contexts = config[2].value.intval; - - - /* initialize the contexts */ - contexts = (pami_context_t *) safemalloc( num_contexts * sizeof(pami_context_t) ); - - result = PAMI_Context_createv(client, NULL, 0, contexts, num_contexts ); - if (my_rank==0) - { - printf("hello world from rank %ld of %ld no. of contexts %ld\n", my_rank, world_size, num_contexts); - fflush(stdout); - } - - - - initialized = 0; - _recv_active = 1; - send_active = 1; - _recv_iteration = 0; - - if (!initialized) - { - for (i=0; i< 1024; i++) - { - conn_list[i][0]= -1; - conn_list[i][1]= -1; - } - nthread_lock_init (&nnti_bgqpami_lock); - nthread_lock_init (&nnti_index_lock); - nthread_lock_init (&nnti_conn_peer_lock); - nthread_lock_init (&nnti_wr_wrhash_lock); - nthread_lock_init (&nnti_buf_bufhash_lock); - pami_dispatch_callback_function dispatch_cb; - dispatch_cb.p2p = dispatch_recv_cb; - pami_dispatch_hint_t dispatch_hint = {0}; - _recv_active = 1; - /* setting this hint to "disable" will disable the pipe_addr!=NULL code path */ - //dispatch_hint.recv_immediate = PAMI_HINT_DISABLE; - result = PAMI_Dispatch_set(contexts[0], dispatch_id, dispatch_cb, (void *)&_recv_active, dispatch_hint); - result = PAMI_Dispatch_set(contexts[1], dispatch_id, dispatch_cb, (void *)&_recv_active, dispatch_hint); - - memset (&transport_global_data, 0, sizeof (bgqpami_transport_global)); - - log_debug (nnti_debug_level, "my_url=%s", my_url); - - - log_debug (nnti_debug_level, "initializing Blue Gene DMA PAMI device"); - - transport_global_data.mypid = getpid (); - transport_global_data.myrank = my_rank; - - - create_peer (&trans_hdl->me, transport_global_data.myrank); - int status = pthread_create(&Progress_thread, NULL, &Progress_function, NULL); - assert(status==0); - - initialized = TRUE; - } - - log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recobgqpamize. - * - */ -NNTI_result_t -NNTI_bgqpami_get_url (const NNTI_transport_t * trans_hdl, - char *url, const uint64_t maxlen) -{ - NNTI_result_t - rc = NNTI_OK; - - assert (trans_hdl); - assert (url); - assert (maxlen > 0); - - log_debug (nnti_debug_level, "enter"); - - strncpy (url, trans_hdl->me.url, maxlen); - url[maxlen - 1] = '\0'; - - log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * - * - * Connectionless transport: parse and populate - * Connected transport: parse, connection and populate - * - */ -NNTI_result_t -NNTI_bgqpami_connect (const NNTI_transport_t * trans_hdl, - const char *url, - const int timeout, NNTI_peer_t * peer_hdl) -{ - NNTI_result_t - rc = NNTI_OK; - - char - transport[NNTI_URL_LEN]; - char - address[NNTI_URL_LEN]; - char * - tmp; - char * - temp; - int - pset_rank; - - bgqpami_connection * - c = NULL; - - - assert (trans_hdl); - assert (peer_hdl); - - log_debug (nnti_debug_level, "enter"); - is_client = 1; - - if (url != NULL) - { - if ((rc = - nnti_url_get_transport (url, transport, NNTI_URL_LEN)) != NNTI_OK) - { - return (rc); - } - if (0 != strcmp (transport, "pami")) - { - /* the peer described by 'url' is not a BGP Torus peer */ - return (NNTI_EINVAL); - } - - if ((rc = nnti_url_get_address (url, address, NNTI_URL_LEN)) != NNTI_OK) - { - return (rc); - } - temp = strdup (url); - tmp = strtok (temp, "//"); - pset_rank = strtol (tmp + 1, NULL, 0); - transport_global_data.remote_rank = pset_rank; /* This is server rank */ - } - else - { - /* */ - rc = NNTI_EINVAL; - goto cleanup; - } - nnti_bgqpami_client_queue_attrs ca_out; - - c = (bgqpami_connection *) calloc (1, sizeof (bgqpami_connection)); - conn_ack = 1; - - client_req_queue_init (c); - memset (&ca_out, 0, sizeof (ca_out)); - ca_out.req_index = c->queue_local_attrs.req_index; - ca_out.req_index_addr = c->queue_local_attrs.req_index_addr; - memcpy (&ca_out.req_index_mem_hdl, - &c->queue_local_attrs.req_index_mem_hdl, sizeof (pami_memregion_t)); - send_once ((char *)&ca_out, sizeof(ca_out), transport_global_data.remote_rank); - insert_conn_rank (pset_rank, c); - - if (c == NULL) - { - rc = NNTI_EIO; - goto cleanup; - } - while (conn_ack > 0){ - PAMI_Context_trylock_advancev(&(contexts[1]), 1, 1000); - } - create_peer (peer_hdl, pset_rank); -cleanup: - log_debug (nnti_debug_level, "exit"); - return (rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - */ -NNTI_result_t -NNTI_bgqpami_disconnect (const NNTI_transport_t * trans_hdl, - NNTI_peer_t * peer_hdl) -{ - NNTI_result_t - rc = NNTI_OK; - - assert (trans_hdl); - assert (peer_hdl); - - log_debug (nnti_debug_level, "enter"); - bgqpami_connection * - conn = - get_conn_rank (peer_hdl->peer.NNTI_remote_process_t_u.bgqpami.pset_rank); - close_connection (conn); - del_conn_rank (peer_hdl->peer.NNTI_remote_process_t_u.bgqpami.pset_rank); - - - log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - */ -NNTI_result_t -NNTI_bgqpami_register_memory (const NNTI_transport_t * trans_hdl, - char *buffer, - const uint64_t size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - const NNTI_peer_t * peer, - NNTI_buffer_t * reg_buf) -{ - NNTI_result_t - rc = NNTI_OK; - - pami_work_request * - wr = NULL; - NNTI_buffer_t * - old_buf = NULL; - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - - - assert (trans_hdl); - assert (buffer); - assert (size > 0); - assert (ops > 0); - assert (reg_buf); - - log_debug (nnti_debug_level, "enter"); - - old_buf = get_buf_bufhash (hash6432shift ((uint64_t) buffer)); - if (old_buf == NULL) - { - bgqpami_mem_hdl = new bgqpami_memory_handle (); - bgqpami_mem_hdl->ref_count = 1; - nthread_lock_init (&bgqpami_mem_hdl->wr_queue_lock); - } - else - { - bgqpami_mem_hdl = (bgqpami_memory_handle *) old_buf->transport_private; - bgqpami_mem_hdl->ref_count++; - } - - assert (bgqpami_mem_hdl); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size = size; - reg_buf->payload = (uint64_t) buffer; - reg_buf->transport_private = (uint64_t) bgqpami_mem_hdl; - - if (peer != NULL) - { - reg_buf->buffer_owner = *peer; - } - - - log_debug (nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - reg_buf->buffer_addr.transport_id = NNTI_TRANSPORT_PAMI; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.size = size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.buf = (uint64_t) buffer; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.owner_rank = - my_rank; - if (bgqpami_mem_hdl->ref_count == 1) - { - if (ops == NNTI_RECV_QUEUE) - { - bgqpami_request_queue_handle * - q_hdl = &transport_global_data.req_queue; - - /* - * This is a receive-only buffer. This buffer is divisible by - * NNTI_REQUEST_BUFFER_SIZE. This buffer can hold more than - * one short request. Assume this buffer is a request queue. - */ - bgqpami_mem_hdl->type = REQUEST_BUFFER; - memset (q_hdl, 0, sizeof (bgqpami_request_queue_handle)); - q_hdl->reg_buf = reg_buf; /* req buffer can be accessed from global req_queue */ - server_req_queue_init (q_hdl, buffer, size, num_elements); - - reg_buf->payload_size = q_hdl->req_size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.type = - NNTI_PAMI_REQUEST_BUFFER; - - } - else if (ops == NNTI_RECV_DST) - { - if (size == NNTI_RESULT_BUFFER_SIZE) - { - /* - * This is a receive-only buffer. This buffer can hold exactly - * one short result. Assume this buffer is a result queue. - */ - bgqpami_mem_hdl->type = RESULT_BUFFER; - - rc = - register_memory (bgqpami_mem_hdl, buffer, - NNTI_RESULT_BUFFER_SIZE, ops); - - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.type = - NNTI_PAMI_RECEIVE_DST; - } - else - { - /* - * This is a receive-only buffer. This buffer doesn't look - * like a request buffer or a result buffer. I don't know - * what it is. Assume it is a regular data buffer. - */ - bgqpami_mem_hdl->type = RECEIVE_BUFFER; - - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.type = - NNTI_PAMI_RECEIVE_DST; - - } - - } - else if (ops == NNTI_SEND_SRC) - { - bgqpami_mem_hdl->type = SEND_BUFFER; - - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - if (rc != NNTI_OK) - { - fprintf (stderr, - "failed registering short request in register memory\n"); - } - - } - else if (ops == NNTI_GET_DST) - { - bgqpami_mem_hdl->type = GET_DST_BUFFER; - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - - } - else if (ops == NNTI_GET_SRC) - { - bgqpami_mem_hdl->type = GET_SRC_BUFFER; - - - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - - } - else if (ops == NNTI_PUT_SRC) - { - bgqpami_mem_hdl->type = PUT_SRC_BUFFER; - - - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - } - else if (ops == NNTI_PUT_DST) - { - bgqpami_mem_hdl->type = PUT_DST_BUFFER; - - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - - } - else if (ops == (NNTI_GET_SRC | NNTI_PUT_DST)) - { - bgqpami_mem_hdl->type = RDMA_TARGET_BUFFER; - - rc = register_memory (bgqpami_mem_hdl, buffer, size, ops); - } - else - { - bgqpami_mem_hdl->type = UNKNOWN_BUFFER; - } - } - wr = (pami_work_request *) calloc (1, sizeof (pami_work_request)); - wr->reg_buf = reg_buf; - wr->last_op = PAMI_OP_REGISTER_RDMA; - register_wc (wr); - - - if (rc == NNTI_OK) - { - reg_buf->buffer_addr.transport_id = NNTI_TRANSPORT_PAMI; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.mem_hdl = (uint64_t)bgqpami_mem_hdl->mem_hdl; - if (bgqpami_mem_hdl->type == REQUEST_BUFFER) - { - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.size = - transport_global_data.req_queue.req_size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.buf = - (uint64_t) transport_global_data.req_queue.req_buffer; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.type = - NNTI_PAMI_REQUEST_BUFFER; - } - else - { - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.size = - reg_buf->payload_size; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.buf = - (uint64_t) reg_buf->payload; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.type = - NNTI_PAMI_SEND_SRC; - - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr = - (uint64_t) & wr->wc; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_mem_hdl = (uint64_t) &wr->wc_mem_hdl; - } - - } - - if (bgqpami_mem_hdl->ref_count == 1) - { - insert_buf_bufhash (reg_buf); - log_debug (nnti_debug_level, "bgqpami_mem_hdl->type==%llu", - (uint64_t) bgqpami_mem_hdl->type); - log_debug (nnti_debug_level, "reg_buf.buf.hash==%llu", - (uint64_t) hash6432shift (reg_buf->buffer_addr. - NNTI_remote_addr_t_u.bgqpami.buf)); - } - - nthread_lock (&bgqpami_mem_hdl->wr_queue_lock); - bgqpami_mem_hdl->wr_queue.push_back (wr); - nthread_unlock (&bgqpami_mem_hdl->wr_queue_lock); - return (rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t -NNTI_bgqpami_unregister_memory (NNTI_buffer_t * reg_buf) -{ - NNTI_result_t - rc = NNTI_OK; - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - - assert (reg_buf); - - log_debug (nnti_debug_level, "enter"); - - bgqpami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - - assert (bgqpami_mem_hdl); - bgqpami_mem_hdl->ref_count--; - - if (bgqpami_mem_hdl->ref_count == 0) - { - log_debug (nnti_debug_level, - "bgqpami_mem_hdl->ref_count is 0. release all resources."); - - if (bgqpami_mem_hdl->type == REQUEST_BUFFER) - { - server_req_queue_destroy (&transport_global_data.req_queue); - - } - else - { - unregister_memory (bgqpami_mem_hdl); - } - - del_buf_bufhash (reg_buf); - nthread_lock (&bgqpami_mem_hdl->wr_queue_lock); - while (!bgqpami_mem_hdl->wr_queue.empty ()) - { - pami_work_request * - wr = bgqpami_mem_hdl->wr_queue.front (); - log_debug (nnti_debug_level, "removing pending wr=%p", wr); - bgqpami_mem_hdl->wr_queue.pop_front (); - del_wr_wrhash (wr); - } - nthread_unlock (&bgqpami_mem_hdl->wr_queue_lock); - } - nthread_lock_fini (&bgqpami_mem_hdl->wr_queue_lock); - if (bgqpami_mem_hdl) - delete - bgqpami_mem_hdl; - - reg_buf->transport_id = NNTI_TRANSPORT_NULL; - reg_buf->payload_size = 0; - reg_buf->payload = 0; - reg_buf->transport_private = 0; - - log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t -NNTI_bgqpami_put (const NNTI_buffer_t * src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t * dest_buffer_hdl, - const uint64_t dest_offset) -{ - bgqpami_memory_handle * - src_buf_mem_hdl = NULL; - NNTI_result_t - rc = NNTI_OK; - pami_work_request * - wr = NULL; - pami_endpoint_t target_ep; - - src_buf_mem_hdl = - (bgqpami_memory_handle *) src_buffer_hdl->transport_private; - size_t - srv_rank = - dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.owner_rank; - wr = (pami_work_request *) calloc (1, sizeof (pami_work_request)); - assert (wr); - wr->wc.local_send_complete = 1; - wr->wc.remote_put_complete = 2; - pami_result_t result = PAMI_Endpoint_create(client, (pami_task_t) srv_rank, 1, &target_ep); - pami_put_simple_t parameters; - parameters.rma.dest = target_ep; - parameters.rma.bytes = src_length; - parameters.rma.cookie = (void *)&wr->wc.remote_put_complete; - parameters.rma.done_fn = cb_done; - parameters.addr.local = (void *)src_buffer_hdl->payload + src_offset; - parameters.addr.remote = (void *)dest_buffer_hdl->buffer_addr. - NNTI_remote_addr_t_u.bgqpami.buf + dest_offset; - parameters.put.rdone_fn = cb_done; - - uint64_t t0 = GetTimeBase(); - - result = PAMI_Put(contexts[4], ¶meters); - - while (wr->wc.remote_put_complete) - { - result = PAMI_Context_trylock_advancev(&(contexts[4]), 1, 1000); - } - - uint64_t t1 = GetTimeBase(); - uint64_t dt = t1-t0; - - register_wc (wr); - wr->reg_buf = src_buffer_hdl; - wr->peer_rank = srv_rank; - memcpy (&wr->wc_dest_mem_hdl, - &dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami. - wc_mem_hdl, sizeof (pami_memregion_t)); - wr->wc_dest_addr = dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr; - - wr->op_state = RDMA_WRITE_INIT; - wr->wc.op = PAMI_OP_PUT_TARGET; - wr->wc.req_rank = transport_global_data.myrank; - wr->wc.byte_len = src_length; - wr->wc.src_offset = src_offset; - wr->wc.dest_offset = dest_offset; - wr->last_op = PAMI_OP_PUT_INITIATOR; - wr->wc.op_complete = 1; - wr->wc.local_send_complete = 2; - wr->wc.dest_addr = dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr; - pami_put_simple_t param; - param.rma.dest = target_ep; - param.rma.bytes = sizeof (nnti_bgqpami_work_completion); - param.rma.cookie = (void *)&wr->wc.local_send_complete; - param.rma.done_fn = cb_done; - param.addr.local = (void *)&wr->wc; - param.addr.remote = (void *)dest_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr; - param.put.rdone_fn = cb_done; - - result = PAMI_Put(contexts[4], ¶m); - - while (wr->wc.local_send_complete) - { - result = PAMI_Context_trylock_advancev(&(contexts[4]), 1, 1000); - } - - nthread_lock (&src_buf_mem_hdl->wr_queue_lock); - src_buf_mem_hdl->wr_queue.push_back (wr); - nthread_unlock (&src_buf_mem_hdl->wr_queue_lock); - insert_wr_wrhash (wr); - log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - */ -NNTI_result_t -NNTI_bgqpami_send (const NNTI_peer_t * peer_hdl, - const NNTI_buffer_t * msg_hdl, - const NNTI_buffer_t * dest_hdl) -{ - NNTI_result_t - rc = NNTI_OK; - - - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - - log_debug (nnti_debug_level, "enter"); - - assert (peer_hdl); - assert (msg_hdl); - - bgqpami_mem_hdl = (bgqpami_memory_handle *) msg_hdl->transport_private; - - if ((dest_hdl == NULL) || (dest_hdl->ops == NNTI_RECV_QUEUE)) - { - bgqpami_connection * - conn = - get_conn_rank (peer_hdl->peer.NNTI_remote_process_t_u.bgqpami. - pset_rank); - assert (conn); - request_send (&conn->queue_local_attrs, - &conn->queue_remote_attrs.server, msg_hdl, - conn->peer_rank); - - } - else - { - rc = NNTI_bgqpami_put (msg_hdl, 0, msg_hdl->payload_size, dest_hdl, 0); - if (rc != NNTI_OK) - log_error (nnti_debug_level, "Put() failed: %d", rc); - } - - log_debug (nnti_debug_level, "sending to (%s)", peer_hdl->url); - - log_debug (nnti_debug_level, "exit"); - - return (rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t -NNTI_bgqpami_get (const NNTI_buffer_t * src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t * dest_buffer_hdl, - const uint64_t dest_offset) -{ - NNTI_result_t - rc = NNTI_OK; - bgqpami_memory_handle * - target_buf_mem_hdl; - int - srv_rank; - pami_work_request * - wr = NULL; - - - wr = (pami_work_request *) calloc (1, sizeof (pami_work_request)); - wr->reg_buf = dest_buffer_hdl; - srv_rank = - src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.owner_rank; - - target_buf_mem_hdl = - (bgqpami_memory_handle *) dest_buffer_hdl->transport_private; - wr->op_state = RDMA_READ_INIT; - wr->wc.op = PAMI_OP_GET_TARGET; - wr->wc.req_rank = srv_rank; - wr->wc.byte_len = src_length; - wr->wc.src_offset = src_offset; - wr->wc.dest_offset = dest_offset; - wr->last_op = PAMI_OP_GET_INITIATOR; - wr->wc.local_get_complete = 1; - wr->peer_rank = srv_rank; - - nthread_lock (&nnti_bgqpami_lock); - pami_endpoint_t target_ep; - pami_result_t result = PAMI_Endpoint_create(client, (pami_task_t) srv_rank, 1, &target_ep); - - pami_get_simple_t parameters; - parameters.rma.dest = target_ep; - //parameters.rma.hints = ; - parameters.rma.bytes = src_length; - parameters.rma.cookie = (void *)&wr->wc.local_get_complete; - parameters.rma.done_fn = cb_done; - parameters.addr.local = (void *)dest_buffer_hdl->payload + dest_offset; - parameters.addr.remote = (void *)src_buffer_hdl->buffer_addr. - NNTI_remote_addr_t_u.bgqpami.buf; - - uint64_t t0 = GetTimeBase(); - - result = PAMI_Get(contexts[5], ¶meters); - - while (wr->wc.local_get_complete) - { - result = PAMI_Context_trylock_advancev(&(contexts[5]), 1, 1000); - } - - uint64_t t1 = GetTimeBase(); - uint64_t dt = t1-t0; - - memcpy (&wr->wc_dest_mem_hdl, - &src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami. - wc_mem_hdl, sizeof (pami_memregion_t)); - wr->wc_dest_addr = src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr; - wr->wc.req_rank = transport_global_data.myrank; - wr->wc.byte_len = src_length; - wr->wc.src_offset = src_offset; - wr->wc.dest_offset = dest_offset; - wr->last_op = PAMI_OP_PUT_INITIATOR; - wr->wc.op_complete = 1; - wr->wc.local_send_complete = 2; - pami_put_simple_t param; - param.rma.dest = target_ep; - param.rma.bytes = sizeof (nnti_bgqpami_work_completion); - param.rma.cookie = (void *)&wr->wc.local_send_complete; - param.rma.done_fn = cb_done; - param.addr.local = (void *)&wr->wc; - param.addr.remote = (void *)src_buffer_hdl->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr; - param.put.rdone_fn = cb_done; - - - result = PAMI_Put(contexts[4], ¶m); - - while (wr->wc.local_send_complete) - { - result = PAMI_Context_trylock_advancev(&(contexts[4]), 1, 1000); - } - - nthread_unlock (&nnti_bgqpami_lock); - nthread_lock (&target_buf_mem_hdl->wr_queue_lock); - target_buf_mem_hdl->wr_queue.push_back (wr); - nthread_unlock (&target_buf_mem_hdl->wr_queue_lock); - insert_wr_wrhash (wr); - log_debug (nnti_debug_level, "exit"); - - return (rc); - -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t -NNTI_bgqpami_wait (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - const int timeout, NNTI_status_t * status) -{ - NNTI_result_t - nnti_rc = NNTI_OK; - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - pami_work_request * - wr = NULL; - bgqpami_connection * - conn = NULL; - int - timeout_per_call; - uint8_t - retry_count = 0; - - - log_debug (nnti_debug_level, "enter"); - - bgqpami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - - if (timeout < 0) - timeout_per_call = MIN_TIMEOUT; - else - timeout_per_call = (timeout < MIN_TIMEOUT) ? MIN_TIMEOUT : timeout; - - retry_count = 0; - wr = bgqpami_mem_hdl->wr_queue.front (); - bgqpami_mem_hdl->wr_queue.pop_front (); - if ((wr->last_op == PAMI_OP_REGISTER_RDMA)) - { - if ((remote_op == NNTI_SEND_SRC) || (remote_op == NNTI_GET_DST) - || (remote_op == NNTI_PUT_SRC)) - { - wr = bgqpami_mem_hdl->wr_queue.front (); - bgqpami_mem_hdl->wr_queue.pop_front (); - } - } - if(wr == NULL){ - fprintf(stderr, "Process events no work request to work on\n"); - return NNTI_OK; - } - nnti_rc = process_event (reg_buf, remote_op, wr, timeout_per_call); - conn = get_conn_rank (wr->peer_rank); - status->op = remote_op; - status->result = nnti_rc; - if (nnti_rc == NNTI_OK) - { - status->start = (uint64_t) reg_buf->payload; - status->offset = wr->wc.byte_offset; - status->length = wr->wc.byte_len; - switch (remote_op) - { - case NNTI_SEND_SRC: - case NNTI_GET_SRC: - case NNTI_PUT_SRC: /* I am client here */ - create_peer (&status->src, transport_global_data.myrank); - create_peer (&status->dest, conn->peer_rank); - // del_wr_wrhash(wr); - break; - case NNTI_RECV_QUEUE: - case NNTI_RECV_DST: - case NNTI_GET_DST: - case NNTI_PUT_DST: - create_peer (&status->src, conn->peer_rank); - create_peer (&status->dest, transport_global_data.myrank); - break; - - } - } - if (nnti_rc == NNTI_OK) - { - - if ((bgqpami_mem_hdl->type == RDMA_TARGET_BUFFER) - || (bgqpami_mem_hdl->type == RECEIVE_BUFFER) - || (bgqpami_mem_hdl->type == GET_SRC_BUFFER) - || (bgqpami_mem_hdl->type == REQUEST_BUFFER) - || (bgqpami_mem_hdl->type == PUT_DST_BUFFER)) - { - bgqpami_mem_hdl->wr_queue.push_back (wr); - } - else - { - del_wr_wrhash (wr); - free (wr); - } - } - - - return (nnti_rc); -} - -/** - * @brief Wait for remote_op on any buffer in buf_list to complete. - * - * Wait for remote_op on any buffer in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the request queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t -NNTI_bgqpami_waitany (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, - uint32_t * which, NNTI_status_t * status) -{ - NNTI_result_t - nnti_rc = NNTI_OK; - - log_level - debug_level = nnti_debug_level; - - log_debug (debug_level, "enter"); - - assert (buf_list); - assert (buf_count > 0); - if (buf_count > 1) - { - /* if there is more than 1 buffer in the list, none of them can be a REQUEST_BUFFER */ - for (uint32_t i = 0; i < buf_count; i++) - { - if (buf_list[i] != NULL) - { - assert (((bgqpami_memory_handle *) buf_list[i]-> - transport_private)->type != REQUEST_BUFFER); - } - } - } - assert (status); - - if (buf_count == 1) - { - nnti_rc = NNTI_bgqpami_wait (buf_list[0], remote_op, timeout, status); - *which = 0; - goto cleanup; - } - - while (1) - { - if (is_any_buf_op_complete (buf_list, buf_count, which) == TRUE) - { - log_debug (debug_level, - "buffer op already complete (which=%u, buf_list[%d]=%p)", - *which, *which, buf_list[*which]); - nnti_rc = NNTI_OK; - } - else - { - log_debug (debug_level, "buffer op NOT complete (buf_list=%p)", - buf_list); - - for (uint32_t i = 0; i < buf_count; i++) - { - if (buf_list[i] != NULL) - { - nnti_rc = - NNTI_bgqpami_wait (buf_list[i], remote_op, - timeout / buf_count, status); - if (nnti_rc == NNTI_OK) - { - *which = i; - break; - } - } - } - - } - } - -cleanup: - - log_debug (debug_level, "exit"); - - trios_stop_timer ("NNTI_bgqpami_waitany", total_time); - - return (nnti_rc); -} - -NNTI_result_t -NNTI_bgqpami_waitall (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, NNTI_status_t ** status) -{ - return NNTI_OK; -} - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t -NNTI_bgqpami_fini (const NNTI_transport_t * trans_hdl) -{ - /* Free up injection counter and global inj fifo and reception FIFO */ - return (NNTI_OK); -} - -static NNTI_result_t -register_wc (pami_work_request * wr) -{ - pami_result_t - pami_result; - size_t - bytes_out; - - -/* This is for Work completion memory handle where ACK is received after PUT/GET */ - pami_result = - PAMI_Memregion_create (contexts[3], &wr->wc, sizeof (nnti_bgqpami_work_completion), &bytes_out, (pami_memregion_t *)wr->wc_mem_hdl); - if (pami_result != PAMI_SUCCESS) - { - fprintf (stderr, - "PAMI memregion create failed in register memory Work completion handle\n"); - return (NNTI_EIO); - } - - log_debug (nnti_debug_level, "exit wr(%p)", wr); - - return ((NNTI_result_t) PAMI_SUCCESS); - -} - - -static - NNTI_result_t -register_memory (bgqpami_memory_handle * hdl, void *buf, uint64_t len, - const NNTI_buf_ops_t remote_op) -{ - NNTI_result_t - rc = NNTI_OK; /* return code */ - pami_result_t - pami_result; - size_t - bytes_out; - - assert (hdl); - pami_result = - PAMI_Memregion_create (contexts[3], buf, len, &bytes_out, &hdl->mem_hdl); - if (pami_result != PAMI_SUCCESS) - fprintf (stderr, "PAMI memregion create failed in register memory \n"); - - return (rc); -} - -static int -unregister_memory (bgqpami_memory_handle * hdl) -{ - int - rc = NNTI_OK; /* return code */ - - log_debug (nnti_debug_level, "exit hdl(%p)", hdl); - - return (rc); -} - - -static - NNTI_result_t -process_event (const NNTI_buffer_t * reg_buf, - const NNTI_buf_ops_t remote_op, - pami_work_request * wr, const int timeout) -{ - NNTI_result_t - nnti_rc = NNTI_OK; - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - - log_level - debug_level = nnti_debug_level; - - bgqpami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - - log_debug (nnti_debug_level, "enter"); - long - entry_time = trios_get_time_ms (); - long - elapsed_time = 0; - - debug_level = nnti_debug_level; - switch (bgqpami_mem_hdl->type) - { - case SEND_BUFFER: - while (wr->wc.remote_put_complete > 0) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[0]), 5, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out SEND BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - } - wr->op_state = SEND_COMPLETE; - wr->wc.op_complete = 1; - break; - case PUT_SRC_BUFFER: - wr->last_op = PAMI_OP_PUT_INITIATOR; - if (wr->op_state == RDMA_WRITE_INIT) - { - while (wr->wc.remote_put_complete > 0) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[0]), 5, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out PUT SRC BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->wc.op_complete = 1; - wr->op_state = RDMA_COMPLETE; - } - break; - case GET_DST_BUFFER: - wr->last_op = PAMI_OP_GET_INITIATOR; - if (wr->op_state == RDMA_READ_INIT) - { - while (wr->wc.local_get_complete > 0) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[1]), 5, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out GET DST BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - wr->wc.op_complete = 1; - } - break; - case REQUEST_BUFFER: - { - uint64_t - index = 0; - bgqpami_request_queue_handle * - q = &transport_global_data.req_queue; - - wr->last_op = PAMI_OP_NEW_REQUEST; - - log_debug (debug_level, - "recv completion - reg_buf=%p current_req_index =%llu processing=%llu", - reg_buf, q->req_index, q->req_processed); - - while (q->req_index == q->req_processed) - { -/* - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out REQUEST BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - */ - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[2]), 1, 1000); - - } - index = q->req_processed; - /* Shyamali wait for work completion buffer to showup */ - while (q->wc_buffer[q->req_processed].req_received != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[3]), 1, 1000); - } - - q->wc_buffer[q->req_processed].byte_offset = index * q->req_size; - q->wc_buffer[q->req_processed].byte_len = q->req_size; - wr->wc = q->wc_buffer[q->req_processed]; - wr->peer_rank = q->wc_buffer[q->req_processed].req_rank; -// fprintf (stderr, "This request came from %d process index %llu\n", -// wr->peer_rank, q->req_processed); - wr->op_state = RECV_COMPLETE; - q->req_processed++; - q->total_req_processed++; - if (q->req_processed > q->req_count) - { - log_error (nnti_debug_level, - "req_processed(%llu) > req_count(%llu) fail", - q->req_processed, q->req_count); - } - if (q->req_processed == (q->req_count / 2)) - { - if (q->req_index >= q->req_count) - { - reset_req_index (q); - log_debug (nnti_debug_level, - "resetting req_processed(%llu) total_req_processed(%llu)", - q->req_processed, q->total_req_processed); - } - else - { - log_debug (nnti_debug_level, - "skipping reset req_processed(%llu) total_req_processed(%llu)", - q->req_processed, q->total_req_processed); - } - } - if (q->req_processed == q->req_processed_reset_limit) - { - q->req_processed = 0; - } - log_debug (nnti_debug_level, - "current req_processed(%llu) req_count(%llu)", - q->req_processed, q->req_count); - } - break; - case RECEIVE_BUFFER: - wr->last_op = PAMI_OP_RECEIVE; - log_debug (debug_level, - "receive buffer - recv completion - reg_buf==%p", reg_buf); - - while (wr->wc.op_complete != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[4]), 1, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out TARGET RECEIVE BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->peer_rank = wr->wc.req_rank; - wr->op_state = RECV_COMPLETE; - break; - case RESULT_BUFFER: - wr->last_op = PAMI_OP_PUT_TARGET; - while (wr->wc.op_complete != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[0]), 5, 1000); - - } - wr->peer_rank = wr->wc.req_rank; - wr->op_state = RDMA_COMPLETE; - break; - case PUT_DST_BUFFER: - wr->last_op = PAMI_OP_PUT_TARGET; - while (wr->wc.op_complete != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[2]), 2, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out PUT DST BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - break; - case GET_SRC_BUFFER: - wr->last_op = PAMI_OP_GET_TARGET; - if (wr->op_state == RDMA_READ_INIT) - { - while (wr->wc.op_complete != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[4]), 2, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out GET SRC BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - } - break; - case RDMA_TARGET_BUFFER: - if ((wr->last_op == PAMI_OP_GET_INITIATOR) || - (wr->last_op == PAMI_OP_PUT_INITIATOR)) - { - - if (wr->op_state == RDMA_TARGET_INIT) - { - while (wr->wc.op_complete != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[2]), 3, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out RDMA TARGET BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - - } - - wr->op_state = RDMA_COMPLETE; - - } - } - else - { - if (wr->op_state == RDMA_TARGET_INIT) - { - - log_debug (debug_level, "RDMA target completion - reg_buf==%p", - reg_buf); - while (wr->wc.op_complete != 1) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[2]), 3, 1000); - elapsed_time = (trios_get_time_ms () - entry_time); - if (((timeout >= 0) && (elapsed_time >= timeout)) - || trios_exit_now ()) - { - log_debug (nnti_debug_level, - "timed out RDMA BUFFER...timeout(%d) elapsed_time(%d) exit_now(%d)", - timeout, elapsed_time, trios_exit_now ()); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - - } - wr->op_state = RDMA_COMPLETE; - - } - } - - break; - case UNKNOWN_BUFFER: - fprintf (stderr, "UNKNOWN_BUFFER\n"); - break; - } - log_debug (nnti_debug_level, "exit"); - return (nnti_rc); -} - - -static void -create_peer (NNTI_peer_t * peer, int rank) -{ - sprintf (peer->url, "pami://%d/", rank); - - peer->peer.transport_id = NNTI_TRANSPORT_PAMI; - peer->peer.NNTI_remote_process_t_u.bgqpami.pset_rank = rank; -} - - - -/* - * Loop over polling recv fifo 0 until everything arrives. - */ -static - NNTI_result_t -recv_full (int rank, void *buf, size_t num) -{ - _recv_active = 1; - while (_recv_active) - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[0]), 1, 1000); - - TRACE_ERR ((stderr, - "recv_full() After advance size recved %d from (%zd)\n", num, - rank)); - memcpy (buf, &_recv_buffer, num); - /* buf = &_recv_buffer; */ - _recv_active = 1; - return NNTI_OK; -} - - -static - NNTI_result_t -new_server_connection (bgqpami_connection * c, int rank) -{ - NNTI_result_t - rc; - bgqpami_request_queue_handle * - q_hdl = &transport_global_data.req_queue; - - /* - * Values passed through DMA MEMFIFO interface to permit initial connection. - */ - struct - { - nnti_bgqpami_server_queue_attrs - server_attrs; - } sa_out; - - - assert (transport_global_data.req_queue.reg_buf); - - - - c->connection_type = SERVER_CONNECTION; - c->peer_rank = rank; - - memset (&sa_out, 0, sizeof (sa_out)); - - sa_out.server_attrs.req_index_addr = - transport_global_data.req_queue.req_index_addr; - - memcpy (&sa_out.server_attrs.req_index_mem_hdl, - &transport_global_data.req_queue.req_index_mem_hdl, - sizeof (pami_memregion_t)); - - sa_out.server_attrs.req_buffer_addr = - (uint64_t) transport_global_data.req_queue.req_buffer; - sa_out.server_attrs.req_size = transport_global_data.req_queue.req_size; - sa_out.server_attrs.req_count = transport_global_data.req_queue.req_count; - memcpy (&sa_out.server_attrs.req_mem_hdl, q_hdl->req_mem_hdl, - sizeof (pami_memregion_t)); - sa_out.server_attrs.wc_buffer_addr = - (uint64_t) transport_global_data.req_queue.wc_buffer; - memcpy (&sa_out.server_attrs.wc_mem_hdl, - &transport_global_data.req_queue.wc_mem_hdl, - sizeof (pami_memregion_t)); - sa_out.server_attrs.unblock_buffer_addr = - (uint64_t) transport_global_data.req_queue.unblock_buffer; - memcpy (&sa_out.server_attrs.unblock_mem_hdl, - &transport_global_data.req_queue.unblock_mem_hdl, - sizeof (pami_memregion_t)); - send_once ((char *)&sa_out, sizeof (sa_out), rank); - -out: - return rc; -} - -static - NNTI_result_t -insert_conn_rank (const uint32_t pset_rank, bgqpami_connection * conn) -{ - NNTI_result_t - rc = NNTI_OK; - - nthread_lock (&nnti_conn_peer_lock); - if (connections_by_rank.find (pset_rank) != connections_by_rank.end ()) - { - conn = connections_by_rank[pset_rank]; - } - connections_by_rank[pset_rank] = conn; // add to connection map - nthread_unlock (&nnti_conn_peer_lock); - - log_debug (nnti_debug_level, "peer connection added (conn=%p)", conn); - - return (rc); -} - -static bgqpami_connection * -get_conn_rank (const uint32_t pset_rank) -{ - - int - key = pset_rank; - bgqpami_connection * - conn = NULL; - - nthread_lock (&nnti_conn_peer_lock); - if (connections_by_rank.find (key) != connections_by_rank.end ()) - { - conn = connections_by_rank[key]; - } - - nthread_unlock (&nnti_conn_peer_lock); - - if (conn != NULL) - { - log_debug (nnti_debug_level, "connection found"); - return conn; - } - - log_debug (nnti_debug_level, "connection NOT found"); - return (NULL); -} - -static bgqpami_connection * -del_conn_rank (const uint32_t pset_rank) -{ - bgqpami_connection * - conn = NULL; - int - key; - - - key = pset_rank; - - nthread_lock (&nnti_conn_peer_lock); - if (connections_by_rank.find (key) != connections_by_rank.end ()) - { - conn = connections_by_rank[key]; - } - - nthread_unlock (&nnti_conn_peer_lock); - - if (conn != NULL) - { - log_debug (nnti_debug_level, "connection found"); - connections_by_rank.erase (key); - } - else - { - log_debug (nnti_debug_level, "connection NOT found"); - } - - return (conn); -} - -/** - * @brief initialize - */ - -/* - * At an explicit BYE message, or at finalize time, shut down a connection. - * If descriptors are posted, defer and clean up the connection structures - * later. - */ -static void -close_connection (bgqpami_connection * c) -{ - - if (c == NULL) - return; - - log_debug (nnti_debug_level, "enter"); - - if (c->connection_type == CLIENT_CONNECTION) - { - client_req_queue_destroy (c); - } - - c->state = DISCONNECTED; - - log_debug (nnti_debug_level, "exit"); -} - -/** - * Check for new connections. The listening socket is left nonblocking - * so this test can be quick; but accept is not really that quick compared - * to polling an Gemini interface, for instance. Returns >0 if an accept worked. - */ -static - NNTI_result_t -check_poll_for_new_connections () -{ - NNTI_result_t - rc = NNTI_OK; - - bgqpami_connection * - conn = NULL; - uint32_t - peer_rank; - int - i; - - _recv_active = 1; - while (_recv_active > 0) - { - pami_result_t result = PAMI_Context_trylock_advancev(&(contexts[1]), 1, 1000); - if (result == PAMI_SUCCESS) - break; - } - nthread_lock (&nnti_bgqpami_lock); - for(i=0; ilast_index_before_reset = value_before_reset; - req_queue_attrs->req_processed_reset_limit = value_before_reset; - req_queue_attrs->req_index = 0; - log_debug (nnti_debug_level, "index before reset(%llu).", - req_queue_attrs->last_index_before_reset); - log_debug (nnti_debug_level, "index after reset(%llu).", - req_queue_attrs->req_index); - - - return (0); -} - -static int -fetch_server_req_buffer_offset (nnti_bgqpami_client_queue_attrs * - local_req_queue_attrs, - nnti_bgqpami_server_queue_attrs * - remote_req_queue_attrs, uint64_t addend, - uint64_t * prev_offset, int srv_rank) -{ - - - - local_req_queue_attrs->req_index = -1; - pami_endpoint_t target_ep; - pami_result_t result = PAMI_Endpoint_create(client, (pami_task_t) srv_rank, 1, &target_ep); - - int active = 1; - uint64_t value; - value = 1; - pami_rmw_t parameters; - parameters.dest = target_ep; - //parameters.hints = ; - parameters.cookie = &active; - parameters.done_fn = cb_done; - parameters.local = (void *)&local_req_queue_attrs->req_index; - parameters.remote = (void *)remote_req_queue_attrs->req_index_addr; - parameters.value = (void *)&value; - parameters.operation = PAMI_ATOMIC_FETCH_ADD; - parameters.type = PAMI_TYPE_UNSIGNED_LONG_LONG; - /* PAMI_ATOMIC_FETCH_ADD : local=remote and remote+=value */ - uint64_t t0 = GetTimeBase(); - - result = PAMI_Rmw(contexts[2], ¶meters); - - while (active) - { - result = PAMI_Context_trylock_advancev(&(contexts[2]), 1, 1000); - } - - uint64_t t1 = GetTimeBase(); - uint64_t dt = t1-t0; - -// fprintf(stderr, " PAMI_Rmw local = %llu in %llu cycles = %lf microseconds \n", local_req_queue_attrs->req_index, (long long unsigned) dt, dt/1600.0 ); - - *prev_offset = local_req_queue_attrs->req_index; - return (0); -} - -static int -send_req (nnti_bgqpami_client_queue_attrs * local_req_queue_attrs, - nnti_bgqpami_server_queue_attrs * remote_req_queue_attrs, - uint64_t offset, const NNTI_buffer_t * reg_buf, int rank, - pami_work_request * wr) -{ - bgqpami_memory_handle * - src_buf_mem_hdl = NULL; - src_buf_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - uint64_t - length; - pami_endpoint_t target_ep; - length = reg_buf->payload_size; - wr->wc.remote_put_complete = 2; /* callback will decr when recv done on remote side */ - wr->wc.local_send_complete = 1; /* First callback returns after injected into local TORUS */ - pami_result_t result = PAMI_Endpoint_create(client, (pami_task_t) rank, 1, &target_ep); - pami_put_simple_t parameters; - parameters.rma.dest = target_ep; - parameters.rma.bytes = length; - parameters.rma.cookie = (void *)&wr->wc.remote_put_complete; - parameters.rma.done_fn = decrement; - parameters.addr.local = (void *)reg_buf->payload; - parameters.addr.remote = (void *)remote_req_queue_attrs->req_buffer_addr + offset; - parameters.put.rdone_fn = decrement; - - uint64_t t0 = GetTimeBase(); -// fprintf(stderr, "buffer address %p remote address %p offset %llu\n", parameters.addr.local, parameters.addr.remote, offset); - - result = PAMI_Put(contexts[3], ¶meters); - while (wr->wc.remote_put_complete) - { - result = PAMI_Context_trylock_advancev(&(contexts[3]), 1, 1000); - } - - uint64_t t1 = GetTimeBase(); - uint64_t dt = t1-t0; - //fprintf(stderr, " send req done %llu cycles = %lf microseconds\n", (long long unsigned)dt, dt/1600.0); - - return (0); -} - -static int -send_req_wc (nnti_bgqpami_client_queue_attrs * client_q, - nnti_bgqpami_server_queue_attrs * server_q, - const NNTI_buffer_t * reg_buf, uint64_t offset, - pami_work_request * wr, int rank) -{ - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - - bgqpami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - pami_endpoint_t target_ep; - wr->wc.remote_put_complete = 2; /* callback will decr when recv done on remote side */ - wr->wc.local_send_complete = 1; /* First callback returns after injected into local TORUS */ - pami_result_t result = PAMI_Endpoint_create(client, (pami_task_t) rank, 1, &target_ep); - pami_put_simple_t parameters; - parameters.rma.dest = target_ep; - parameters.rma.bytes = sizeof (nnti_bgqpami_work_completion); - parameters.rma.cookie = (void *)&wr->wc.remote_put_complete; - parameters.rma.done_fn = cb_done; - parameters.addr.local = (void *)&wr->wc; - parameters.addr.remote = (void *)server_q->wc_buffer_addr + offset; - parameters.put.rdone_fn = cb_done; - - uint64_t t0 = GetTimeBase(); - - result = PAMI_Put(contexts[3], ¶meters); - - while (wr->wc.remote_put_complete) - { - result = PAMI_Context_trylock_advancev(&(contexts[3]), 1, 1000); - } - -// fprintf(stderr, "Send request wc completed \n"); - return (0); - -} - - - -static int -request_send (nnti_bgqpami_client_queue_attrs * client_q, - nnti_bgqpami_server_queue_attrs * server_q, - const NNTI_buffer_t * reg_buf, int rank) -{ - int - rc = 0; - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - uint64_t - offset; - pami_work_request * - wr = NULL; - - uint32_t - wc_size = sizeof (nnti_bgqpami_work_completion); - - bgqpami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - - assert (bgqpami_mem_hdl); - wr = (pami_work_request *) malloc (sizeof (pami_work_request)); - memset (wr, 0, sizeof (pami_work_request)); - assert (wr); - - wr->reg_buf = reg_buf; - - bgqpami_mem_hdl->wr_queue.push_back (wr); - - insert_wr_wrhash (wr); - - log_debug (nnti_debug_level, "enter"); - - log_debug (nnti_debug_level, "calling fetch_add_buffer_offset()"); - rc = fetch_server_req_buffer_offset (client_q, server_q, 1, &offset, rank); - - log_debug (nnti_debug_level, "calling send_req()"); - rc = - send_req (client_q, server_q, offset * server_q->req_size, reg_buf, rank, - wr); - if (rc != NNTI_OK) - log_error (nnti_debug_level, "send_req() failed: %d", rc); - wr->wc.op = PAMI_OP_SEND_REQ; - wr->wc.req_received = 1; - wr->wc.req_rank = transport_global_data.myrank; - wr->wc.byte_len = reg_buf->payload_size; - wr->wc.byte_offset = client_q->req_index * server_q->req_size; - wr->wc.src_offset = 0; - wr->wc.dest_offset = client_q->req_index * server_q->req_size; - wr->peer_rank = rank; - -/* Shyamali */ - log_debug (nnti_debug_level, "calling send_wc()"); - rc = send_req_wc (client_q, server_q, reg_buf, offset * wc_size, wr, rank); - /* Shyamali END */ - log_debug (nnti_debug_level, "exit"); - return (0); -} - - - -static int -client_req_queue_init (bgqpami_connection * c) -{ - int - rc = 0; - size_t - bytes; - - nnti_bgqpami_client_queue_attrs * - q = &c->queue_local_attrs; - q->req_index = 0; - q->req_index_addr = (uint64_t) & q->req_index; - PAMI_Memregion_create (&q->req_index_mem_hdl, - (void *) q->req_index_addr, sizeof (uint64_t), &bytes, - (pami_memregion_t *)&q->req_index_mem_hdl); - return (rc); -} - -static int -client_req_queue_destroy (bgqpami_connection * c) -{ - return (0); -} - -static int -server_req_queue_init (bgqpami_request_queue_handle * q, - char *buffer, uint64_t req_size, uint64_t req_count) -{ - int - i; - size_t - bytes; - uint64_t - unblock_size; - bgqpami_memory_handle * - bgqpami_mem_hdl = (bgqpami_memory_handle *) q->reg_buf->transport_private; - - - q->req_buffer = buffer; - q->req_size = req_size; - q->req_count = req_count; - q->req_buffer_size = req_count * req_size; - - q->wc_buffer_size = q->req_count * sizeof (nnti_bgqpami_work_completion); - q->wc_buffer = - (nnti_bgqpami_work_completion *) calloc (q->req_count, - sizeof - (nnti_bgqpami_work_completion)); - - unblock_size = MAX_CONNECTION * sizeof (nnti_bgqpami_work_completion); - q->unblock_buffer = - (nnti_bgqpami_fetch_index_req *) calloc (MAX_CONNECTION, - sizeof - (nnti_bgqpami_fetch_index_req)); - PAMI_Memregion_create (contexts[2], buffer, q->req_buffer_size, &bytes, &q->req_mem_hdl); - PAMI_Memregion_create (contexts[2], q->wc_buffer, q->wc_buffer_size,&bytes, &q->wc_mem_hdl); - PAMI_Memregion_create (contexts[2],q->unblock_buffer, unblock_size, &bytes, &q->unblock_mem_hdl); - q->req_index = 0; - q->req_index_addr = (uint64_t) & q->req_index; - - q->req_processed = 0; - q->req_processed_reset_limit = q->req_count; - bgqpami_mem_hdl->type = REQUEST_BUFFER; - PAMI_Memregion_create (contexts[2], (void *) q->req_index_addr, sizeof(uint64_t), &bytes, &q->req_index_mem_hdl); - for (i = 0; i < MAX_CONNECTION; ++i) - q->unblock_buffer[i].req_rank = -1; - - start_connection_listener_thread (); - return 0; -} - -static int -server_req_queue_destroy (bgqpami_request_queue_handle * q) -{ - return 0; -} - -void -dump_packet (char *buf) -{ - - char * - pkt = buf; - int - i; - for (i = 0; i < 1200; i++) - fprintf (stderr, "%c", pkt[i]); - fprintf (stderr, "\n"); -} - -static NNTI_result_t -insert_buf_bufhash (NNTI_buffer_t * buf) -{ - NNTI_result_t - rc = NNTI_OK; - uint32_t - h = - hash6432shift ((uint64_t) buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami. - buf); - - nthread_lock (&nnti_buf_bufhash_lock); - assert (buffers_by_bufhash.find (h) == buffers_by_bufhash.end ()); - buffers_by_bufhash[h] = buf; - nthread_unlock (&nnti_buf_bufhash_lock); - - log_debug (nnti_debug_level, "bufhash buffer added (buf=%p)", buf); - - return (rc); -} - -static NNTI_buffer_t * -get_buf_bufhash (const uint32_t bufhash) -{ - NNTI_buffer_t * - buf = NULL; - - log_debug (nnti_debug_level, "looking for bufhash=%llu", - (uint64_t) bufhash); - nthread_lock (&nnti_buf_bufhash_lock); - if (buffers_by_bufhash.find (bufhash) != buffers_by_bufhash.end ()) - { - buf = buffers_by_bufhash[bufhash]; - } - nthread_unlock (&nnti_buf_bufhash_lock); - - if (buf != NULL) - { - log_debug (nnti_debug_level, "buffer found (buf=%p)", buf); - return buf; - } - - log_debug (nnti_debug_level, "buffer NOT found"); - print_bufhash_map (); - - return (NULL); -} - -static NNTI_buffer_t * -del_buf_bufhash (NNTI_buffer_t * buf) -{ - uint32_t - h = - hash6432shift ((uint64_t) buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami. - buf); - log_level - debug_level = nnti_debug_level; - - nthread_lock (&nnti_buf_bufhash_lock); - if (buffers_by_bufhash.find (h) != buffers_by_bufhash.end ()) - { - buf = buffers_by_bufhash[h]; - } - nthread_unlock (&nnti_buf_bufhash_lock); - - if (buf != NULL) - { - log_debug (debug_level, "buffer found"); - buffers_by_bufhash.erase (h); - } - else - { - log_debug (debug_level, "buffer NOT found"); - } - - return (buf); -} - -static void -print_bufhash_map () -{ - if (!logging_debug (nnti_debug_level)) - { - return; - } - - buf_by_bufhash_iter_t - i; - for (i = buffers_by_bufhash.begin (); i != buffers_by_bufhash.end (); i++) - { - log_debug (nnti_debug_level, "bufhash_map key=%llu buf=%p", i->first, - i->second); - } -} - -static NNTI_result_t -insert_wr_wrhash (pami_work_request * wr) -{ - NNTI_result_t - rc = NNTI_OK; - uint32_t - h = hash6432shift ((uint64_t) wr); - - nthread_lock (&nnti_wr_wrhash_lock); - assert (wr_by_wrhash.find (h) == wr_by_wrhash.end ()); - wr_by_wrhash[h] = wr; - nthread_unlock (&nnti_wr_wrhash_lock); - - log_debug (nnti_debug_level, - "wrhash work request added (wr=%p ; wr.hash=%llu)", wr, - (uint64_t) h); - - return (rc); -} - -static pami_work_request * -get_wr_wrhash (const uint32_t wrhash) -{ - pami_work_request * - wr = NULL; - - log_debug (nnti_debug_level, "looking for wrhash=%llu", (uint64_t) wrhash); - nthread_lock (&nnti_wr_wrhash_lock); - if (wr_by_wrhash.find (wrhash) != wr_by_wrhash.end ()) - { - wr = wr_by_wrhash[wrhash]; - } - nthread_unlock (&nnti_wr_wrhash_lock); - - if (wr != NULL) - { - log_debug (nnti_debug_level, "work request found (wr=%p)", wr); - return wr; - } - - log_debug (nnti_debug_level, "work request NOT found"); - print_wrhash_map (); - - return (NULL); -} - -static pami_work_request * -del_wr_wrhash (pami_work_request * wr) -{ - uint32_t - h = hash6432shift ((uint64_t) wr); - log_level - debug_level = nnti_debug_level; - - nthread_lock (&nnti_wr_wrhash_lock); - if (wr_by_wrhash.find (h) != wr_by_wrhash.end ()) - { - wr = wr_by_wrhash[h]; - } - nthread_unlock (&nnti_wr_wrhash_lock); - - if (wr != NULL) - { - log_debug (debug_level, "work request found"); - wr_by_wrhash.erase (h); - } - else - { - log_debug (debug_level, "work request NOT found"); - } - - return (wr); -} - -static void -print_wrhash_map () -{ - if (!logging_debug (nnti_debug_level)) - { - return; - } - - wr_by_wrhash_iter_t - i; - for (i = wr_by_wrhash.begin (); i != wr_by_wrhash.end (); i++) - { - log_debug (nnti_debug_level, "wrhash_map key=%llu wr=%p", i->first, - i->second); - } -} - - -static int8_t -is_any_buf_op_complete (const NNTI_buffer_t ** buf_list, - const uint32_t buf_count, uint32_t * which) -{ - int8_t - rc = FALSE; - - log_debug (nnti_debug_level, "enter"); - - for (uint32_t i = 0; i < buf_count; i++) - { - if ((buf_list[i] != NULL) && - (is_wr_queue_empty (buf_list[i]) == FALSE) && - (is_buf_op_complete (buf_list[i]) == TRUE)) - { - - *which = i; - rc = TRUE; - break; - } - } - - log_debug (nnti_debug_level, "exit (rc=%d)", rc); - - return (rc); -} - -static int8_t -is_wr_queue_empty (const NNTI_buffer_t * reg_buf) -{ - int8_t - rc = FALSE; - bgqpami_memory_handle * - pami_mem_hdl = NULL; - - log_debug (nnti_debug_level, "enter"); - - pami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - assert (pami_mem_hdl); - - if (pami_mem_hdl->wr_queue.empty ()) - { - rc = TRUE; - } - - log_debug (nnti_debug_level, "exit (rc=%d)", rc); - return (rc); -} - - -static int8_t -is_buf_op_complete (const NNTI_buffer_t * reg_buf) -{ - int8_t - rc = FALSE; - bgqpami_memory_handle * - pami_mem_hdl = NULL; - pami_work_request * - wr = NULL; - - log_debug (nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - pami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - - if (is_wr_queue_empty (reg_buf) == TRUE) - { - log_debug (nnti_debug_level, - "work request queue is empty - return FALSE"); - rc = FALSE; - } - else - { - wr = pami_mem_hdl->wr_queue.front (); - assert (wr); - - rc = is_wr_complete (wr); - } - - if (rc == TRUE) - { - log_debug (nnti_debug_level, "op is complete"); - } - log_debug (nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return (rc); -} - -static int8_t -is_wr_complete (pami_work_request * wr) -{ - int8_t - rc = FALSE; - bgqpami_memory_handle * - pami_mem_hdl = NULL; - - pami_mem_hdl = (bgqpami_memory_handle *) wr->reg_buf->transport_private; - assert (pami_mem_hdl); - - switch (pami_mem_hdl->type) - { - case SEND_BUFFER: - if (wr->op_state == SEND_COMPLETE) - { - rc = TRUE; - } - break; - case PUT_SRC_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case GET_DST_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case REQUEST_BUFFER: - case RECEIVE_BUFFER: - if (wr->op_state == RECV_COMPLETE) - { - rc = TRUE; - } - break; - case PUT_DST_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case GET_SRC_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case RDMA_TARGET_BUFFER: - if (wr->op_state == RDMA_COMPLETE) - { - rc = TRUE; - } - break; - case UNKNOWN_BUFFER: - break; - } - - log_debug (nnti_debug_level, "exit (rc=%d)", rc); - return (rc); -} - -static NNTI_result_t -repost_recv_work_request (NNTI_buffer_t * reg_buf, pami_work_request * wr) -{ - bgqpami_memory_handle * - bgqpami_mem_hdl = NULL; - - log_debug (nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - bgqpami_mem_hdl = (bgqpami_memory_handle *) reg_buf->transport_private; - assert (bgqpami_mem_hdl); - - memset (&wr->wc, 0, sizeof (nnti_bgqpami_work_completion)); - - wr->last_op = 0; - - memset (&wr->op_state, 0, sizeof (bgqpami_op_state_t)); - register_wc (wr); - - bgqpami_mem_hdl->wr_queue.push_back (wr); - - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_addr = - (uint64_t) & wr->wc; - reg_buf->buffer_addr.NNTI_remote_addr_t_u.bgqpami.wc_mem_hdl = (uint64_t)wr->wc_mem_hdl; - log_debug (nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return (NNTI_OK); -} diff --git a/packages/trios/libraries/nessie/nnti/nnti_pami.h b/packages/trios/libraries/nessie/nnti/nnti_pami.h deleted file mode 100644 index 8c788bef1fef..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_pami.h +++ /dev/null @@ -1,144 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_bgqpami.h - * - * Created on: Apr 4, 2012 - * Author: thkorde - */ - -#ifndef NNTI_PAMI_H_ -#define NNTI_PAMI_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "Trios_config.h" - -#include "Trios_nnti.h" -#include "nnti_internal.h" - - - - -NNTI_result_t NNTI_bgqpami_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_bgqpami_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -NNTI_result_t NNTI_bgqpami_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_bgqpami_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_bgqpami_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - const NNTI_peer_t *peer, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_bgqpami_unregister_memory ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_bgqpami_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl); - -NNTI_result_t NNTI_bgqpami_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset); - -NNTI_result_t NNTI_bgqpami_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset); - -NNTI_result_t NNTI_bgqpami_wait ( - const NNTI_buffer_t *reg_buf, - const NNTI_buf_ops_t remote_op, - const int timeout, - NNTI_status_t *status); - -NNTI_result_t NNTI_bgqpami_waitany ( - const NNTI_buffer_t **buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -NNTI_result_t NNTI_bgqpami_waitall ( - const NNTI_buffer_t **buf_list, - const uint32_t buf_count, - const NNTI_buf_ops_t remote_op, - const int timeout, - NNTI_status_t **status); - -NNTI_result_t NNTI_bgqpami_fini ( - const NNTI_transport_t *trans_hdl); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_PAMI_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/nnti_ptls.cpp b/packages/trios/libraries/nessie/nnti/nnti_ptls.cpp deleted file mode 100644 index 1f52bb5ac1d6..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_ptls.cpp +++ /dev/null @@ -1,3197 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_ptls.c - * - * Created on: Jan 13, 2011 - * Author: thkorde - */ - -#include "Trios_config.h" -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_nnti_fprint_types.h" - -// MPI is only used to increment the PID -#ifdef HAVE_TRIOS_MPI -#include -#endif - -#include -#include - -#include -#include - -#include "nnti_ptls.h" -#include "nnti_utils.h" - - - -/* if defined, the RDMA initiator will send an ACK message to the RDMA - * target when the RDMA op is complete. the target process must wait - * on the target buffer in order to get the ACK. this creates two-sided - * semantics for RDMA ops. in this mode, when the wait returns the - * the RDMA op is complete and status indicates what data was addressed. - */ -#define USE_RDMA_TARGET_ACK -/* if undefined, the ACK message is NOT sent to the RDMA target when - * the RDMA op is complete. this creates one-sided semantics for RDMA - * ops. in this mode, the target has no idea when the RDMA op is - * complete and what data was addressed. NNTI_wait() returns NNTI_EINVAL - * if passed a target buffer. - */ -#undef USE_RDMA_TARGET_ACK - - - -#define PTL_OP_PUT_INITIATOR 1 -#define PTL_OP_GET_INITIATOR 2 -#define PTL_OP_PUT_TARGET 3 -#define PTL_OP_GET_TARGET 4 -#define PTL_OP_SEND 5 -#define PTL_OP_NEW_REQUEST 6 -#define PTL_OP_RECEIVE 8 - -/** - * @brief Track the state of a PtlPut (i'm the initiator). - */ -typedef struct { - int send_start; - int send_end; - int ack; - int unlink; -} ptl_put_initiator_state_t; -/** - * @brief Track the state of a PtlGet (i'm the initiator). - */ -typedef struct { - int send_start; - int send_end; - int reply_start; - int reply_end; - int unlink; -} ptl_get_initiator_state_t; -/** - * @brief Track the state of a PtlPut (i'm the target). - */ -typedef struct { - int put_start; - int put_end; -} ptl_put_target_state_t; -/** - * @brief Track the state of a PtlGet (i'm the target). - */ -typedef struct { - int get_start; - int get_end; -} ptl_get_target_state_t; - -typedef union { - ptl_put_initiator_state_t put_initiator; - ptl_get_initiator_state_t get_initiator; - ptl_put_target_state_t put_target; - ptl_get_target_state_t get_target; -} ptl_op_state_t; - -typedef enum { - REQUEST_BUFFER, - RECEIVE_BUFFER, - SEND_BUFFER, - GET_SRC_BUFFER, - GET_DST_BUFFER, - PUT_SRC_BUFFER, - PUT_DST_BUFFER, - RDMA_TARGET_BUFFER, - UNKNOWN_BUFFER -} ptl_buffer_type; - -typedef struct portals_work_request { - NNTI_buffer_t *reg_buf; - NNTI_peer_t peer; - uint64_t src_offset; - uint64_t dst_offset; - uint64_t length; - - ptl_event_t last_event; - - uint8_t last_op; - ptl_op_state_t op_state; - uint8_t is_last_op_complete; - - ptl_handle_me_t me_h; - ptl_md_t md; - ptl_handle_md_t md_h; -} portals_work_request; - -typedef std::deque wr_queue_t; -typedef std::deque::iterator wr_queue_iter_t; - -typedef struct portals_memory_handle { - ptl_buffer_type type; - - ptl_handle_eq_t eq_h; - ptl_pt_index_t buffer_id; - ptl_process_id_t match_id; - ptl_match_bits_t match_bits; - ptl_match_bits_t ignore_bits; - ptl_handle_me_t me_h; - ptl_md_t md; - ptl_handle_md_t md_h; - - wr_queue_t wr_queue; -} portals_memory_handle; - - -#define NUM_REQ_QUEUES 2 -typedef struct portals_request_queue_handle { - NNTI_buffer_t *reg_buf; - - /* incoming queues */ - char *req_queue[NUM_REQ_QUEUES]; - - /* each message is no larger than req_size */ - int req_size; - - /* each md can recv reqs_per_queue messages */ - int reqs_per_queue; - - /* keep track of the queue index and count */ - int indices[NUM_REQ_QUEUES]; - int queue_count[NUM_REQ_QUEUES]; - - /* for each queue, we need these structs */ - ptl_handle_me_t me_h[NUM_REQ_QUEUES]; - ptl_md_t md[NUM_REQ_QUEUES]; - ptl_handle_md_t md_h[NUM_REQ_QUEUES]; -} portals_request_queue_handle; - -typedef struct portals_transport_global { - - ptl_handle_ni_t ni_h; - - ptl_process_id_t me; - - ptl_handle_eq_t req_eq_h; - ptl_handle_eq_t data_eq_h; - - portals_request_queue_handle req_queue; - - bool init_called_mpi_init; - -} portals_transport_global; - - - -static nthread_lock_t nnti_ptl_lock; - - -static void set_req_pid(NNTI_pid *pid); -static const NNTI_work_request_t *decode_event_wr( - const NNTI_work_request_t *wait_wr, - const ptl_event_t *event); -static int process_event( - const NNTI_work_request_t *reg_buf, - const ptl_event_t *event); -//static NNTI_result_t post_recv_work_request( -// NNTI_buffer_t *reg_buf); -//static NNTI_result_t repost_recv_work_request( -// NNTI_work_request_t *wr); -static int8_t is_wr_complete( - portals_work_request *wr); -static int8_t is_any_wr_complete( - portals_work_request **wr_list, - const uint32_t wr_count, - uint32_t *which); -static int8_t is_all_wr_complete( - portals_work_request **wr_list, - const uint32_t wr_count); -static int8_t is_wr_complete( - NNTI_work_request_t *wr); -static int8_t is_any_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - uint32_t *which); -static int8_t is_all_wr_complete( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); -//static portals_work_request *first_incomplete_wr( -// portals_memory_handle *ptl_mem_hdl); -//static int8_t is_wr_queue_empty( -// const NNTI_buffer_t *reg_buf); -//static int8_t is_buf_op_complete( -// const NNTI_buffer_t *reg_buf); -//static int8_t is_any_buf_op_complete( -// const NNTI_buffer_t **buf_list, -// const uint32_t buf_count, -// uint32_t *which); -//static int8_t is_all_buf_ops_complete( -// const NNTI_buffer_t **buf_list, -// const uint32_t buf_count); - -static void create_status( - const NNTI_work_request_t *reg_buf, - int nnti_rc, - NNTI_status_t *status); -static void create_peer( - NNTI_peer_t *peer, - ptl_nid_t nid, - ptl_pid_t pid); - - -#define PTL_MEM_HDL(b) ((portals_memory_handle *)((b)->transport_private)) -#define PTL_WORK_REQUEST(wr) ((portals_work_request *)((wr)->transport_private)) - -static bool ptl_initialized=false; - - -static portals_transport_global transport_global_data; -static const int MIN_TIMEOUT = 10; /* in milliseconds */ - -/** - * @brief Initialize NNTI to use a specific transport. - * - * Enable the use of a particular transport by this process. my_url - * allows the process to have some control (if possible) over the - * URL assigned for the transport. For example, a Portals URL to put - * might be "ptl://-1,128". This would tell Portals to use the default - * network ID, but use PID=128. If the transport - * can be initialized without this info (eg. a Portals client), my_url can - * be NULL or empty. - */ -NNTI_result_t NNTI_ptl_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - int max_interfaces; - ptl_ni_limits_t actual; - - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; -// char memdesc[NNTI_URL_LEN]; - char *sep; - -// NNTI_nid nid; - NNTI_pid pid=-1; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - - - if (!ptl_initialized) { - - nthread_lock_init(&nnti_ptl_lock); - - if (my_url != NULL) { - if ((nnti_rc=nnti_url_get_transport(my_url, transport, NNTI_URL_LEN)) != NNTI_OK) { - return(nnti_rc); - } - if (0!=strcmp(transport, "ptl")) { - return(NNTI_EINVAL); - } - - if ((nnti_rc=nnti_url_get_address(my_url, address, NNTI_URL_LEN)) != NNTI_OK) { - return(nnti_rc); - } - - sep=strchr(address, ':'); -// nid=strtol(address, NULL, 0); - pid=strtol(sep+1, NULL, 0); - } - if (pid == -1) { - set_req_pid(&pid); - } - - - log_debug(nnti_debug_level, "initializing portals"); - -// /* The UTCP NAL requires that the application defines where the Portals -// * API and library should send any output. We'll send the output to -// * the same file as logger. -// */ -// utcp_api_out = logger_get_file(); -// utcp_lib_out = logger_get_file(); -// -// -// /* register trace groups (let someone else enable) */ -// trace_counter_gid = trace_get_gid(TRACE_RPC_COUNTER_GNAME); -// trace_interval_gid = trace_get_gid(TRACE_RPC_INTERVAL_GNAME); - - - memset(&transport_global_data, 0, sizeof(portals_transport_global)); - - /* initialize the portals library */ - log_debug(nnti_debug_level, "initializing portals library"); - rc = PtlInit(&max_interfaces); - if (rc) { - log_fatal(nnti_debug_level,"PtlInit() failed, %s", ptl_err_str[rc]); - abort(); - } - - /* initialize the portals interface */ - log_debug(nnti_debug_level, "initializing portals network interface - pid=%d", (int)pid); - rc = PtlNIInit(PTL_IFACE_DEFAULT, pid, NULL, &actual, &transport_global_data.ni_h); - if ((rc != PTL_OK) && (rc != PTL_IFACE_DUP)) { - log_fatal(nnti_debug_level, "PtlNIInit() failed, %s", ptl_err_str[rc]); - abort(); - } - - ptl_process_id_t ptl_id; - rc = PtlGetId(transport_global_data.ni_h, &ptl_id); - if (rc != PTL_OK) { - log_error(nnti_debug_level,"failed %s", ptl_err_str[rc]); - } - - transport_global_data.me.nid = ptl_id.nid; - transport_global_data.me.pid = ptl_id.pid; - - /* create an event queue */ - /* TODO: should we share an event queue? */ - nthread_lock(&nnti_ptl_lock); - rc = PtlEQAlloc( - transport_global_data.ni_h, - 5000, - PTL_EQ_HANDLER_NONE, - &transport_global_data.data_eq_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != NNTI_OK) { - log_error(nnti_debug_level, "failed to allocate eventq"); - abort(); - } - log_debug(nnti_debug_level, "allocated transport_global_data data eq=%d", transport_global_data.data_eq_h); - - if (logging_info(nnti_debug_level)) { - fprintf(logger_get_file(), "Portals Initialized: nid=%llu, pid=%llu\n", - (unsigned long long)transport_global_data.me.nid, - (unsigned long long)transport_global_data.me.pid); - } - - log_debug(nnti_debug_level, "sizeof(trans_hdl)=%d", trans_hdl); - - create_peer(&trans_hdl->me, transport_global_data.me.nid, transport_global_data.me.pid); - - ptl_initialized = true; - } - - - log_debug(nnti_debug_level, "exit"); - - - return(nnti_rc); -} - - -/** - * @brief Return the URL field of this transport. - * - * Return the URL field of this transport. After initialization, the transport will - * have a specific location on the network where peers can contact it. The - * transport will convert this location to a string that other instances of the - * transport will recognize. - * - * URL format: "transport://address/memory_descriptor" - * - transport - (required) identifies how the URL should parsed - * - address - (required) uniquely identifies a location on the network - * - ex. "ptl://nid:pid/", "ib://ip_addr:port" - * - memory_descriptor - (optional) transport-specific representation of RMA params - */ -NNTI_result_t NNTI_ptl_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - assert(trans_hdl); - assert(url); - assert(maxlen>0); - - strncpy(url, trans_hdl->me.url, maxlen); - url[maxlen-1]='\0'; - - return(nnti_rc); -} - - -/** - * @brief Prepare for communication with the peer identified by url. - * - * Parse url in a transport specific way. Perform any transport specific - * actions necessary to begin communication with this peer. - * -// * If the peer is found and responds -// * to a ping, a handle will be allocated and assigned to the pointer. This -// * handle should be used to move data to/from the peer. - * - * Connectionless transport: parse and populate - * Connected transport: parse, connection and populate - * - */ -NNTI_result_t NNTI_ptl_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - char transport[NNTI_URL_LEN]; - char address[NNTI_URL_LEN]; -// char memdesc[NNTI_URL_LEN]; - char *sep; - - NNTI_nid nid; - NNTI_pid pid; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(peer_hdl); - - if (url != NULL) { - if ((nnti_rc=nnti_url_get_transport(url, transport, NNTI_URL_LEN)) != NNTI_OK) { - return(nnti_rc); - } - if (0!=strcmp(transport, "ptl")) { - /* the peer described by 'url' is not a Portals peer */ - return(NNTI_EINVAL); - } - - if ((nnti_rc=nnti_url_get_address(url, address, NNTI_URL_LEN)) != NNTI_OK) { - return(nnti_rc); - } - - sep=strchr(address, ':'); - nid=strtol(address, NULL, 0); - pid=strtol(sep+1, NULL, 0); - } else { - /* */ - return(NNTI_EINVAL); - } - - create_peer( - peer_hdl, - nid, - pid); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Terminate communication with this peer. - * - * Perform any transport specific actions necessary to end communication with - * this peer. - */ -NNTI_result_t NNTI_ptl_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - assert(trans_hdl); - assert(peer_hdl); - - return(nnti_rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_ptl_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - char *buf=(char *)malloc(element_size*num_elements); - assert(buf); - - nnti_rc=NNTI_ptl_register_memory( - trans_hdl, - buf, - element_size, - num_elements, - ops, - reg_buf); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_ptl_alloc", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_ptl_free ( - NNTI_buffer_t *reg_buf) -{ - NNTI_result_t nnti_rc=NNTI_OK; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - char *buf=NNTI_BUFFER_C_POINTER(reg_buf); - assert(buf); - - nnti_rc=NNTI_ptl_unregister_memory(reg_buf); - - free(buf); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Prepare a block of memory for network operations. - * - * Wrap a user allocated block of memory in an NNTI_buffer_t. The transport - * may take additional actions to prepare the memory for network send/receive. - * If the memory block doesn't meet the transport's requirements for memory - * regions, then errors or poor performance may result. - */ -NNTI_result_t NNTI_ptl_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - static uint64_t mbits=1; - - portals_memory_handle *ptl_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(trans_hdl); - assert(buffer); - assert(element_size>0); - assert(num_elements>0); - assert(ops>0); - assert(reg_buf); - - ptl_mem_hdl=new portals_memory_handle(); - assert(ptl_mem_hdl); - - reg_buf->transport_id = trans_hdl->id; - reg_buf->buffer_owner = trans_hdl->me; - reg_buf->ops = ops; - reg_buf->payload_size = element_size; - reg_buf->payload = (uint64_t)buffer; - reg_buf->transport_private = (uint64_t)ptl_mem_hdl; - - log_debug(nnti_debug_level, "rpc_buffer->payload_size=%ld", - reg_buf->payload_size); - - ptl_mem_hdl->eq_h=PTL_EQ_NONE; - ptl_mem_hdl->me_h=0; - ptl_mem_hdl->md_h=0; - - ptl_mem_hdl->match_id.nid = PTL_NID_ANY; - ptl_mem_hdl->match_id.pid = PTL_PID_ANY; - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val=(NNTI_remote_addr_t *)calloc(1, sizeof(NNTI_remote_addr_t)); - reg_buf->buffer_segments.NNTI_remote_addr_array_t_len=1; - - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id = reg_buf->transport_id; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.size = element_size; - - /* - * Buffer types are divided into four groups here. - * Request Buffer - This buffer type requires immediate registration. - * Events are generated, so an EQ is assigned. - * Receive Buffer - This buffer type requires immediate registration. - * Events are generated, so an EQ is assigned. - * Send/RDMA Initiators - This buffer type allows for lazy registration. - * The buffer will be register when the operation - * is initiated. Events are generated, so an EQ - * is assigned. - * RDMA Targets - This buffer type requires immediate registration. - * Events are NOT generated, so an EQ is NOT assigned. - */ - - if (ops == NNTI_RECV_QUEUE) { - uint32_t index=0; - portals_request_queue_handle *q_hdl=&transport_global_data.req_queue; - - ptl_mem_hdl->type=REQUEST_BUFFER; - - ptl_mem_hdl->buffer_id = NNTI_REQ_PT_INDEX; - ptl_mem_hdl->match_bits = 0; - ptl_mem_hdl->ignore_bits = 0; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id = (NNTI_portals_indices)ptl_mem_hdl->buffer_id; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits = ptl_mem_hdl->match_bits; - - q_hdl->reg_buf=reg_buf; - - q_hdl->req_size=element_size; - q_hdl->reqs_per_queue=num_elements/NUM_REQ_QUEUES; - - /* create an event queue */ - nthread_lock(&nnti_ptl_lock); - rc = PtlEQAlloc( - transport_global_data.ni_h, - num_elements*2, - PTL_EQ_HANDLER_NONE, - &transport_global_data.req_eq_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "PtlEQAlloc() failed"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - ptl_mem_hdl->eq_h=transport_global_data.req_eq_h; - log_debug(nnti_debug_level, "allocated eq=%d", ptl_mem_hdl->eq_h); - - for (index=0; indexindices[index] = index; - q_hdl->queue_count[index] = 0; - - /* allocate the buffer for the incoming MD */ - q_hdl->req_queue[index] = buffer + (index*q_hdl->reqs_per_queue*q_hdl->req_size); - - /* initialize the buffer */ - memset(q_hdl->req_queue[index], 0, q_hdl->reqs_per_queue*q_hdl->req_size); - - /* initialize the MD */ - memset(&q_hdl->md[index], 0, sizeof(ptl_md_t)); - q_hdl->md[index].start = q_hdl->req_queue[index]; - q_hdl->md[index].length = q_hdl->reqs_per_queue*q_hdl->req_size; - q_hdl->md[index].threshold = q_hdl->reqs_per_queue; - q_hdl->md[index].max_size = q_hdl->req_size; - q_hdl->md[index].options = PTL_MD_OP_PUT | PTL_MD_MAX_SIZE; - q_hdl->md[index].user_ptr = &q_hdl->indices[index]; /* used to store the index */ - q_hdl->md[index].eq_handle = ptl_mem_hdl->eq_h; - - log_debug(nnti_debug_level, "attaching match entry to index=%d", - ptl_mem_hdl->buffer_id); - - /* Attach the match entry to the portal index */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_RETAIN, - PTL_INS_AFTER, - &q_hdl->me_h[index]); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "could not attach ME"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - - /* Attach the MD to the match entry */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - q_hdl->me_h[index], - q_hdl->md[index], - PTL_RETAIN, - &q_hdl->md_h[index]); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "could not alloc eq: %s", - ptl_err_str[rc]); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "attached q_hdl->md_h[%d]: %d", index, q_hdl->md_h[index]); - - reg_buf->payload_size=q_hdl->req_size; - } - -// post_recv_work_request(reg_buf); - - } else if (ops == NNTI_RECV_DST) { - - ptl_mem_hdl->type=RECEIVE_BUFFER; - - ptl_mem_hdl->buffer_id = NNTI_RECV_PT_INDEX; - ptl_mem_hdl->match_bits = mbits++; - ptl_mem_hdl->ignore_bits = 0; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id = (NNTI_portals_indices)ptl_mem_hdl->buffer_id; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits = ptl_mem_hdl->match_bits; - - ptl_mem_hdl->eq_h=transport_global_data.data_eq_h; - - /* create a match entry (unlink with MD) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_UNLINK, - PTL_INS_AFTER, - &ptl_mem_hdl->me_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach me"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "allocated me=%d with bufid=%d, match_id(%d,%d), mbits=%d", - ptl_mem_hdl->me_h, ptl_mem_hdl->buffer_id, ptl_mem_hdl->match_id.nid, ptl_mem_hdl->match_id.pid, ptl_mem_hdl->match_bits); - - /* initialize the md */ - memset(&ptl_mem_hdl->md, 0, sizeof(ptl_md_t)); - ptl_mem_hdl->md.start = buffer; - ptl_mem_hdl->md.length = element_size; - ptl_mem_hdl->md.threshold = PTL_MD_THRESH_INF; - ptl_mem_hdl->md.options = PTL_MD_OP_PUT|PTL_MD_OP_GET|PTL_MD_MANAGE_REMOTE|PTL_MD_TRUNCATE; - ptl_mem_hdl->md.user_ptr = reg_buf; - ptl_mem_hdl->md.eq_handle = ptl_mem_hdl->eq_h; - - /* attach the memory descriptor (manually unlink) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - ptl_mem_hdl->me_h, - ptl_mem_hdl->md, - PTL_RETAIN, - &ptl_mem_hdl->md_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach md"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "attached ptl_mem_hdl->md_h: %d", ptl_mem_hdl->md_h); - -// post_recv_work_request(reg_buf); - - } else if ((ops == NNTI_SEND_SRC) || - (ops == NNTI_GET_DST) || - (ops == NNTI_PUT_SRC)) { - - /* - * These are initiator buffers. - */ - - if (ops == NNTI_SEND_SRC) { - ptl_mem_hdl->type=SEND_BUFFER; - } else if (ops == NNTI_GET_DST) { - ptl_mem_hdl->type=GET_DST_BUFFER; - } else if (ops == NNTI_PUT_SRC) { - ptl_mem_hdl->type=PUT_SRC_BUFFER; - } - - ptl_mem_hdl->buffer_id = NNTI_DATA_PT_INDEX; - ptl_mem_hdl->match_bits = mbits++; - ptl_mem_hdl->ignore_bits = 0; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id = (NNTI_portals_indices)ptl_mem_hdl->buffer_id; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits = ptl_mem_hdl->match_bits; - - ptl_mem_hdl->eq_h=transport_global_data.data_eq_h; - - } else if ((ops == NNTI_GET_SRC) || - (ops == NNTI_PUT_DST) || - (ops == (NNTI_GET_SRC|NNTI_PUT_DST))) { - - /* - * These are eventless target buffers. - */ - - if (ops == NNTI_GET_SRC) { - ptl_mem_hdl->type=GET_SRC_BUFFER; - } else if (ops == NNTI_PUT_DST) { - ptl_mem_hdl->type=PUT_DST_BUFFER; - } else if (ops == (NNTI_GET_SRC|NNTI_PUT_DST)) { - ptl_mem_hdl->type=RDMA_TARGET_BUFFER; - } - - ptl_mem_hdl->buffer_id = NNTI_DATA_PT_INDEX; - ptl_mem_hdl->match_bits = mbits++; - ptl_mem_hdl->ignore_bits = 0; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id = (NNTI_portals_indices)ptl_mem_hdl->buffer_id; - reg_buf->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits = ptl_mem_hdl->match_bits; - -#if defined(USE_RDMA_TARGET_ACK) - ptl_mem_hdl->eq_h = transport_global_data.data_eq_h; - post_recv_work_request(reg_buf); -#else - ptl_mem_hdl->eq_h = PTL_EQ_NONE; -#endif - - /* create a match entry (unlink with MD) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_UNLINK, - PTL_INS_AFTER, - &ptl_mem_hdl->me_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach me"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "allocated me=%d with bufid=%d, match_id(%d,%d), mbits=%d", - ptl_mem_hdl->me_h, ptl_mem_hdl->buffer_id, ptl_mem_hdl->match_id.nid, ptl_mem_hdl->match_id.pid, ptl_mem_hdl->match_bits); - - /* initialize the md */ - memset(&ptl_mem_hdl->md, 0, sizeof(ptl_md_t)); - ptl_mem_hdl->md.start = buffer; - ptl_mem_hdl->md.length = element_size; - ptl_mem_hdl->md.threshold = PTL_MD_THRESH_INF; - ptl_mem_hdl->md.options = PTL_MD_OP_PUT|PTL_MD_OP_GET|PTL_MD_MANAGE_REMOTE|PTL_MD_TRUNCATE; - ptl_mem_hdl->md.user_ptr = reg_buf; - ptl_mem_hdl->md.eq_handle = ptl_mem_hdl->eq_h; - - /* attach the memory descriptor (manually unlink) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - ptl_mem_hdl->me_h, - ptl_mem_hdl->md, - PTL_RETAIN, - &ptl_mem_hdl->md_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach md"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "attached ptl_mem_hdl->md_h: %d", ptl_mem_hdl->md_h); - } - - - -cleanup: - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "reg_buf", - "end of NNTI_ptl_register_memory", reg_buf); - } - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Prepare a list of memory segments for network operations. - * - * Wrap a list of user allocated memory segments in an NNTI_buffer_t. The - * transport may take additional actions to prepare the memory segments for - * network send/receive. If the memory segments don't meet the transport's - * requirements for memory regions, then errors or poor performance may - * result. - * - */ -NNTI_result_t NNTI_ptl_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf) -{ - return NNTI_OK; -} - - -/** - * @brief Cleanup after network operations are complete. - * - * Destroy an NNTI_buffer_t that was previously created by NNTI_regsiter_buffer(). - * It is the user's responsibility to release the the memory region. - */ -NNTI_result_t NNTI_ptl_unregister_memory ( - NNTI_buffer_t *reg_buf) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - portals_memory_handle *ptl_mem_hdl=NULL; - log_level debug_level = nnti_debug_level; - - log_debug(nnti_debug_level, "enter"); - - assert(reg_buf); - - ptl_mem_hdl=PTL_MEM_HDL(reg_buf); - - assert(ptl_mem_hdl); - - log_debug(nnti_debug_level, "unregistering reg_buf(%p) buf(%p) md_h(%d) eq_h(%d)", - reg_buf, reg_buf->payload, ptl_mem_hdl->md_h, ptl_mem_hdl->eq_h); - - if (reg_buf->ops == NNTI_RECV_QUEUE) { - uint32_t index=0; - portals_request_queue_handle *q_hdl=&transport_global_data.req_queue; - - for (index=0; indexmd_h[%d]: %d", index, q_hdl->md_h[index]); - nthread_lock(&nnti_ptl_lock); - rc = PtlMDUnlink(q_hdl->md_h[index]); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_warn(debug_level, "unable to unlink memory descriptor for queue %d: %s", - index, ptl_err_str[rc]); - nnti_rc = NNTI_ENOMEM; - goto cleanup; - } - } - - /* free the event queue */ - log_debug(debug_level, "freeing ptl_mem_hdl->eq_h: %d", ptl_mem_hdl->eq_h); - nthread_lock(&nnti_ptl_lock); - rc = PtlEQFree(transport_global_data.req_eq_h); - nthread_unlock(&nnti_ptl_lock); - transport_global_data.req_eq_h=PTL_EQ_NONE; - if (rc != PTL_OK) { - log_fatal(debug_level, "unable to free EQ: %s", ptl_err_str[rc]); - nnti_rc = NNTI_ENOMEM; - goto cleanup; - } - } else if ((reg_buf->ops == NNTI_RECV_DST) || - (reg_buf->ops == NNTI_GET_SRC) || - (reg_buf->ops == NNTI_PUT_DST) || - (reg_buf->ops == (NNTI_GET_SRC|NNTI_PUT_DST))) { - - log_debug(debug_level, "unlinking ptl_mem_hdl->md_h: %d", ptl_mem_hdl->md_h); - nthread_lock(&nnti_ptl_lock); - rc = PtlMDUnlink(ptl_mem_hdl->md_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(debug_level, "failed to unlink MD: %s", ptl_err_str[rc]); - nnti_rc = NNTI_ENOMEM; - goto cleanup; - } - - ptl_mem_hdl->eq_h=PTL_EQ_NONE; - } - - -cleanup: - - if (ptl_mem_hdl) - delete ptl_mem_hdl; - if (reg_buf->buffer_segments.NNTI_remote_addr_array_t_val) - free(reg_buf->buffer_segments.NNTI_remote_addr_array_t_val); - - reg_buf->transport_id = NNTI_TRANSPORT_NULL; - PORTALS_SET_MATCH_ANY(®_buf->buffer_owner); - reg_buf->ops = (NNTI_buf_ops_t)0; - reg_buf->payload_size = 0; - reg_buf->payload = 0; - reg_buf->transport_private = 0; - - log_debug(debug_level, "Finished unregistering, rc=%d",rc); - - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Send a message to a peer. - * - * Send a message (msg_hdl) to a peer (peer_hdl). It is expected that the - * message is small, but the exact maximum size is transport dependent. - */ -NNTI_result_t NNTI_ptl_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr=NULL; - ptl_process_id_t dest_id; - ptl_pt_index_t buffer_id; - ptl_match_bits_t match_bits; - - log_debug(nnti_debug_level, "enter"); - - assert(peer_hdl); - assert(msg_hdl); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "msg_hdl", - "NNTI_ptl_send", msg_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_hdl", - "NNTI_ptl_send", dest_hdl); - } - - ptl_mem_hdl=PTL_MEM_HDL(msg_hdl); - assert(ptl_mem_hdl); - ptl_wr=(portals_work_request *)calloc(1, sizeof(portals_work_request)); - assert(ptl_wr); - - /* create a match entry (unlink with MD) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_UNLINK, - PTL_INS_AFTER, - &ptl_mem_hdl->me_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach me"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "allocated me=%d with bufid=%d, match_id(%d,%d), mbits=%d", - ptl_mem_hdl->me_h, ptl_mem_hdl->buffer_id, ptl_mem_hdl->match_id.nid, ptl_mem_hdl->match_id.pid, ptl_mem_hdl->match_bits); - - /* initialize the md */ - memset(&ptl_mem_hdl->md, 0, sizeof(ptl_md_t)); - ptl_mem_hdl->md.start =(void*)msg_hdl->payload; - ptl_mem_hdl->md.length =msg_hdl->payload_size; - ptl_mem_hdl->md.threshold=2; - ptl_mem_hdl->md.options =PTL_MD_OP_PUT|PTL_MD_OP_GET|PTL_MD_MANAGE_REMOTE|PTL_MD_TRUNCATE; - ptl_mem_hdl->md.user_ptr =wr; - ptl_mem_hdl->md.eq_handle=ptl_mem_hdl->eq_h; - - /* attach the memory descriptor (manually unlink) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - ptl_mem_hdl->me_h, - ptl_mem_hdl->md, - PTL_UNLINK, - &ptl_mem_hdl->md_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach md"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - - - memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); - - if (dest_hdl == NULL) { - ptl_wr->peer=*peer_hdl; - dest_id.nid =peer_hdl->peer.NNTI_remote_process_t_u.portals.nid; - dest_id.pid =peer_hdl->peer.NNTI_remote_process_t_u.portals.pid; - buffer_id =NNTI_REQ_PT_INDEX; - match_bits =0; - } else { - ptl_wr->peer=dest_hdl->buffer_owner; - dest_id.nid =dest_hdl->buffer_owner.peer.NNTI_remote_process_t_u.portals.nid; - dest_id.pid =dest_hdl->buffer_owner.peer.NNTI_remote_process_t_u.portals.pid; - buffer_id =dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id; - match_bits =dest_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits; - } - - ptl_wr->reg_buf =(NNTI_buffer_t *)msg_hdl; - ptl_wr->src_offset=0; - ptl_wr->dst_offset=0; - ptl_wr->length =msg_hdl->payload_size; - - log_debug(nnti_debug_level, "sending to (nid=%d, pid=%d, buffer_id=%d, mbits=%d)", dest_id.nid, dest_id.pid, buffer_id, match_bits); - - rc=PtlPut( - ptl_mem_hdl->md_h, - PTL_ACK_REQ, - dest_id, - buffer_id, - 0, - match_bits, - 0, - 0); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to send with PUT: %s", ptl_err_str[rc]); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - ptl_wr->last_op=PTL_OP_SEND; - - wr->transport_id =msg_hdl->transport_id; - wr->ops =NNTI_SEND_SRC; - wr->transport_private=(uint64_t)ptl_wr; - - ptl_mem_hdl->wr_queue.push_back(wr); - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Transfer data to a peer. - * - * Put the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_ptl_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr=NULL; - ptl_process_id_t dest_id; - - log_debug(nnti_debug_level, "enter"); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "src_buffer_hdl", - "NNTI_ptl_put", src_buffer_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_buffer_hdl", - "NNTI_ptl_put", dest_buffer_hdl); - } - - ptl_mem_hdl=PTL_MEM_HDL(src_buffer_hdl); - assert(ptl_mem_hdl); - ptl_wr=(portals_work_request *)calloc(1, sizeof(portals_work_request)); - assert(ptl_wr); - - /* create a match entry (unlink with MD) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_UNLINK, - PTL_INS_AFTER, - &ptl_mem_hdl->me_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach me"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "allocated me=%d with bufid=%d, match_id(%d,%d), mbits=%d", - ptl_mem_hdl->me_h, ptl_mem_hdl->buffer_id, ptl_mem_hdl->match_id.nid, ptl_mem_hdl->match_id.pid, ptl_mem_hdl->match_bits); - - /* initialize the md */ - memset(&ptl_mem_hdl->md, 0, sizeof(ptl_md_t)); - ptl_mem_hdl->md.start = (void*)src_buffer_hdl->payload; - ptl_mem_hdl->md.length = src_buffer_hdl->payload_size; - ptl_mem_hdl->md.threshold = 2; - ptl_mem_hdl->md.options = PTL_MD_OP_PUT|PTL_MD_OP_GET|PTL_MD_MANAGE_REMOTE|PTL_MD_TRUNCATE; - ptl_mem_hdl->md.user_ptr = wr; - ptl_mem_hdl->md.eq_handle = ptl_mem_hdl->eq_h; - - /* attach the memory descriptor (manually unlink) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - ptl_mem_hdl->me_h, - ptl_mem_hdl->md, - PTL_UNLINK, - &ptl_mem_hdl->md_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach md"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - - - ptl_wr->reg_buf =(NNTI_buffer_t *)src_buffer_hdl; - ptl_wr->peer =dest_buffer_hdl->buffer_owner; - ptl_wr->src_offset=src_offset; - ptl_wr->dst_offset=dest_offset; - ptl_wr->length =src_length; - ptl_wr->last_op =PTL_OP_PUT_INITIATOR; - - memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); - - dest_id.nid=dest_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.portals.nid; - dest_id.pid=dest_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.portals.pid; - - rc=PtlPutRegion( - ptl_mem_hdl->md_h, - src_offset, - src_length, - PTL_ACK_REQ, - dest_id, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id, - 0, - dest_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits, - dest_offset, - 0); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to PUT region: %s", ptl_err_str[rc]); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - log_debug(nnti_debug_level, "putting to (%s, eq=%d)", dest_buffer_hdl->buffer_owner.url, ptl_mem_hdl->eq_h); - - wr->transport_id =src_buffer_hdl->transport_id; - wr->ops =NNTI_PUT_SRC; - wr->transport_private=(uint64_t)ptl_wr; - - ptl_mem_hdl->wr_queue.push_back(wr); - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Transfer data from a peer. - * - * Get the contents of src_buffer_hdl into dest_buffer_hdl. It is - * assumed that the destination is at least src_length bytes in size. - * - */ -NNTI_result_t NNTI_ptl_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr) -{ - int rc=0; - NNTI_result_t nnti_rc=NNTI_OK; - - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr=NULL; - ptl_process_id_t src_id; - - log_debug(nnti_debug_level, "enter"); - - assert(src_buffer_hdl); - assert(dest_buffer_hdl); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "src_buffer_hdl", - "NNTI_ptl_get", src_buffer_hdl); - } - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "dest_buffer_hdl", - "NNTI_ptl_get", dest_buffer_hdl); - } - - log_debug(nnti_debug_level, "getting from (%s, src_offset=%llu, src_length=%llu, dest_offset=%llu)", - src_buffer_hdl->buffer_owner.url, src_offset, src_length, dest_offset); - - if (logging_debug(nnti_debug_level)) { - fprint_NNTI_buffer(logger_get_file(), "src_buffer_hdl", - "NNTI_ptl_get", src_buffer_hdl); - fprint_NNTI_buffer(logger_get_file(), "dest_buffer_hdl", - "NNTI_ptl_get", dest_buffer_hdl); - } - - ptl_mem_hdl=PTL_MEM_HDL(dest_buffer_hdl); - assert(ptl_mem_hdl); - ptl_wr=(portals_work_request *)calloc(1, sizeof(portals_work_request)); - assert(ptl_wr); - - /* create a match entry (unlink with MD) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_UNLINK, - PTL_INS_AFTER, - &ptl_mem_hdl->me_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach me"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - log_debug(nnti_debug_level, "allocated me=%d with bufid=%d, match_id(%d,%d), mbits=%d", - ptl_mem_hdl->me_h, ptl_mem_hdl->buffer_id, ptl_mem_hdl->match_id.nid, ptl_mem_hdl->match_id.pid, ptl_mem_hdl->match_bits); - - /* initialize the md */ - memset(&ptl_mem_hdl->md, 0, sizeof(ptl_md_t)); - ptl_mem_hdl->md.start = (void*)dest_buffer_hdl->payload; - ptl_mem_hdl->md.length = dest_buffer_hdl->payload_size; - ptl_mem_hdl->md.threshold = 1; - ptl_mem_hdl->md.options = PTL_MD_OP_PUT|PTL_MD_OP_GET|PTL_MD_MANAGE_REMOTE|PTL_MD_TRUNCATE; - ptl_mem_hdl->md.user_ptr = wr; - ptl_mem_hdl->md.eq_handle = ptl_mem_hdl->eq_h; - - /* attach the memory descriptor (manually unlink) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - ptl_mem_hdl->me_h, - ptl_mem_hdl->md, - PTL_UNLINK, - &ptl_mem_hdl->md_h); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to attach md"); - nnti_rc=NNTI_ENOMEM; - goto cleanup; - } - - - ptl_wr->reg_buf =(NNTI_buffer_t *)dest_buffer_hdl; - ptl_wr->peer =src_buffer_hdl->buffer_owner; - ptl_wr->src_offset=src_offset; - ptl_wr->dst_offset=dest_offset; - ptl_wr->length =src_length; - ptl_wr->last_op =PTL_OP_GET_INITIATOR; - - - memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); - - src_id.nid=src_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.portals.nid; - src_id.pid=src_buffer_hdl->buffer_owner.peer.NNTI_remote_process_t_u.portals.pid; - - rc=PtlGetRegion( - ptl_mem_hdl->md_h, - dest_offset, - src_length, - src_id, - src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id, - 0, - src_buffer_hdl->buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits, - src_offset); - if (rc != PTL_OK) { - log_error(nnti_debug_level, "failed to GET region: %s", ptl_err_str[rc]); - nnti_rc = NNTI_EBADRPC; - goto cleanup; - } - - log_debug(nnti_debug_level, "getting from (%s, eq=%d)", src_buffer_hdl->buffer_owner.url, ptl_mem_hdl->eq_h); - - wr->transport_id =dest_buffer_hdl->transport_id; - wr->ops =NNTI_GET_DST; - wr->transport_private=(uint64_t)ptl_wr; - - ptl_mem_hdl->wr_queue.push_back(wr); - -cleanup: - log_debug(nnti_debug_level, "exit"); - - return(nnti_rc); -} - - -/** - * @brief Transfer data to a peer. - * - * \param[in] src_buffer_hdl A buffer containing the data to put. - * \param[in] src_length The number of bytes to put. - * \param[in] dest_buffer_list A list of buffers to put the data into. - * \param[in] dest_count The number of destination buffers. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_ptl_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr) -{ - return NNTI_OK; -} - - -/** - * @brief Transfer data from a peer. - * - * \param[in] src_buffer_list A list of buffers containing the data to get. - * \param[in] src_length The number of bytes to get. - * \param[in] src_count The number of source buffers. - * \param[in] dest_buffer_hdl A buffer to get the data into. - * \return A result code (NNTI_OK or an error) - */ -NNTI_result_t NNTI_ptl_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr) -{ - return NNTI_OK; -} - - -NNTI_result_t NNTI_ptl_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_ptl_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_ptl_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr) -{ - return NNTI_ENOTSUP; -} - - -NNTI_result_t NNTI_ptl_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr) -{ - return NNTI_ENOTSUP; -} - - -/** - * @brief Create a receive work request that can be used to wait for buffer - * operations to complete. - * - */ -NNTI_result_t NNTI_ptl_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr) -{ - portals_work_request *ptl_wr=NULL; - portals_memory_handle *ptl_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); - - ptl_mem_hdl=PTL_MEM_HDL(reg_buf); - assert(ptl_mem_hdl); - - ptl_wr=(portals_work_request *)calloc(1, sizeof(portals_work_request)); - assert(ptl_wr); - ptl_wr->reg_buf = reg_buf; - - memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); - - wr->transport_id =reg_buf->transport_id; - wr->reg_buf =reg_buf; - wr->ops =reg_buf->ops; - wr->transport_private=(uint64_t)ptl_wr; - - ptl_mem_hdl->wr_queue.push_back(wr); - - log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from a previous receive - * and prepares it for reuse. - * - */ -NNTI_result_t NNTI_ptl_clear_work_request ( - NNTI_work_request_t *wr) -{ - portals_work_request *ptl_wr=NULL; - portals_memory_handle *ptl_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - ptl_wr=PTL_WORK_REQUEST(wr); - assert(ptl_wr); - ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); - assert(ptl_mem_hdl); - - memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); - - ptl_mem_hdl->wr_queue.push_back(wr); - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Disassociates a receive work request from reg_buf. - * - */ -NNTI_result_t NNTI_ptl_destroy_work_request ( - NNTI_work_request_t *wr) -{ - portals_work_request *ptl_wr=NULL; - portals_memory_handle *ptl_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - ptl_wr=PTL_WORK_REQUEST(wr); - assert(ptl_wr); - ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); - assert(ptl_mem_hdl); - - free(ptl_wr); - - log_debug(nnti_debug_level, "exit (wr=%p)", wr); - - return(NNTI_OK); -} - - -/** - * @brief Attempts to cancel an NNTI opertion. - * - */ -NNTI_result_t NNTI_ptl_cancel ( - NNTI_work_request_t *wr) -{ - return NNTI_OK; -} - - -/** - * @brief Attempts to cancel a list of NNTI opertions. - * - */ -NNTI_result_t NNTI_ptl_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count) -{ - return NNTI_OK; -} - - -/** - * @brief Interrupts NNTI_wait*() - * - */ -NNTI_result_t NNTI_ptl_interrupt ( - const NNTI_transport_t *trans_hdl) -{ - char dummy=0xAA; - - log_debug(nnti_debug_level, "enter"); - - log_debug(nnti_debug_level, "exit"); - - return NNTI_OK; -} - - -/** - * @brief Wait for remote_op on reg_buf to complete. - * - * Wait for remote_op on reg_buf to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - */ -NNTI_result_t NNTI_ptl_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status) -{ - int rc=PTL_OK; - NNTI_result_t nnti_rc=NNTI_OK; - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr=NULL; - -// const NNTI_buffer_t *reg_buf=NULL; - - const NNTI_work_request_t *wait_wr=NULL; - - int elapsed_time=0; - int timeout_per_call; - ptl_event_t event; - int which_eq=0; - - log_level debug_level=nnti_debug_level; - - trios_declare_timer(call_time); - trios_declare_timer(total_time); - - trios_start_timer(total_time); - - log_debug(debug_level, "enter"); - - assert(wr); - assert(status); - - ptl_wr=PTL_WORK_REQUEST(wr); - assert(ptl_wr); - ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); - assert(ptl_mem_hdl); -// ptl_wr=first_incomplete_wr(ptl_mem_hdl); -// assert(ptl_wr); - - if (ptl_mem_hdl->type == REQUEST_BUFFER) { - memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); - } - - if (is_wr_complete(ptl_wr) == TRUE) { - log_debug(debug_level, "work request already complete"); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "work request NOT complete"); - - timeout_per_call = MIN_TIMEOUT; - - while (1) { - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - return NNTI_ECANCELED; - } - - log_debug(debug_level, "waiting on wr(%p) eq_h(%d)", wr, ptl_mem_hdl->eq_h); - - memset(&event, 0, sizeof(ptl_event_t)); - log_debug(debug_level, "lock before poll"); - trios_start_timer(call_time); - // nthread_lock(&nnti_ptl_lock); - rc = PtlEQPoll(&ptl_mem_hdl->eq_h, 1, timeout_per_call, &event, &which_eq); - // nthread_unlock(&nnti_ptl_lock); - trios_stop_timer("NNTI_ptl_wait - PtlEQPoll", call_time); - log_debug(debug_level, "polling status is %s", ptl_err_str[rc]); - - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\ttype = %d", event.type); - log_debug(debug_level, "\tinitiator = (%llu, %llu)", (unsigned long long)event.initiator.nid, (unsigned long long)event.initiator.pid); - log_debug(debug_level, "\tuid = %d", event.uid); - log_debug(debug_level, "\tjid = %d", event.jid); - log_debug(debug_level, "\tpt_index = %d", event.pt_index); - log_debug(debug_level, "\tmatch_bits = %d", event.match_bits); - log_debug(debug_level, "\trlength = %llu", (unsigned long long)event.rlength); - log_debug(debug_level, "\tmlength = %llu", (unsigned long long)event.mlength); - log_debug(debug_level, "\toffset = %llu", (unsigned long long)event.offset); - log_debug(debug_level, "\tmd_handle = %d", event.md_handle); - log_debug(debug_level, "\tmd.start = %p", event.md.start); - log_debug(debug_level, "\tmd.length = %d", event.md.length); - log_debug(debug_level, "\tmd.max_size = %d", event.md.max_size); - log_debug(debug_level, "\tmd.threshold = %d", event.md.threshold); - log_debug(debug_level, "\tmd.user_ptr = %p", event.md.user_ptr); - log_debug(debug_level, "}"); - - - /* case 1: success */ - if (rc == PTL_OK) { - nnti_rc = NNTI_OK; - } - /* case 2: success, but some events were dropped */ - else if (rc == PTL_EQ_DROPPED) { - log_warn(debug_level, "PtlEQPoll dropped some events"); - log_warn(debug_level, "PtlEQPoll succeeded, but at least one event was dropped"); - nnti_rc = NNTI_OK; - } - /* case 3: timed out */ - else if (rc == PTL_EQ_EMPTY) { - elapsed_time += timeout_per_call; - - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout > 0) && (elapsed_time >= timeout))) { - log_debug(debug_level, "PtlEQPoll timed out: %s", - ptl_err_str[rc]); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - /* continue if the timeout has not expired */ - /* log_debug(debug_level, "timedout... continuing"); */ - - - - continue; - } - /* case 4: failure */ - else { - log_error(debug_level, "PtlEQPoll failed (eq_handle[%d]==%d): %s", - which_eq, ptl_mem_hdl->eq_h, ptl_err_str[rc]); - nnti_rc = NNTI_EIO; - break; - } - - wait_wr=decode_event_wr(wr, &event); - process_event(wait_wr, &event); - - if (is_wr_complete(ptl_wr) == TRUE) { - break; - } - } - } - - create_status(wr, nnti_rc, status); - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_ptl_wait", status); - } - - if ((nnti_rc==NNTI_OK) && (ptl_mem_hdl->buffer_id == NNTI_REQ_PT_INDEX)) { - portals_request_queue_handle *q_hdl=&transport_global_data.req_queue; - - int index = *(int *)ptl_wr->last_event.md.user_ptr; - /* get the index of the queue */ - q_hdl->queue_count[index]++; - - log_debug(debug_level, "queue_count[%d]: %d", index, q_hdl->queue_count[index]); - - /* if we've processed all we can on this queue, reset */ - if (q_hdl->queue_count[index] >= q_hdl->reqs_per_queue) { - - log_debug(debug_level, "Resetting MD on queue[%d]", index); - - /* Unlink the ME (also unlinks the MD) */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEUnlink(q_hdl->me_h[index]); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(debug_level, "Could not unlink ME: %s", ptl_err_str[rc]); - goto cleanup; - } - - /* Re-attach the match-list entry */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMEAttach( - transport_global_data.ni_h, - ptl_mem_hdl->buffer_id, - ptl_mem_hdl->match_id, - ptl_mem_hdl->match_bits, - ptl_mem_hdl->ignore_bits, - PTL_RETAIN, - PTL_INS_AFTER, - &q_hdl->me_h[index]); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(debug_level, "Could not reset ME: %s", ptl_err_str[rc]); - goto cleanup; - } - - /* Re-attach the MD */ - nthread_lock(&nnti_ptl_lock); - rc = PtlMDAttach( - q_hdl->me_h[index], - q_hdl->md[index], - PTL_RETAIN, - &q_hdl->md_h[index]); - nthread_unlock(&nnti_ptl_lock); - if (rc != PTL_OK) { - log_error(debug_level, "Could not reset MD: %s", ptl_err_str[rc]); - goto cleanup; - } - - q_hdl->queue_count[index] = 0; - } - } - - if (nnti_rc==NNTI_OK) { - ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); - assert(ptl_mem_hdl); -// ptl_wr=ptl_mem_hdl->wr_queue.front(); -// assert(ptl_wr); -// ptl_mem_hdl->wr_queue.pop_front(); - - switch (ptl_mem_hdl->type) { - case REQUEST_BUFFER: - case RECEIVE_BUFFER: -#if defined(USE_RDMA_TARGET_ACK) - case GET_SRC_BUFFER: - case PUT_DST_BUFFER: - case RDMA_TARGET_BUFFER: -#endif -// repost_recv_work_request(wr); - break; - case SEND_BUFFER: - case GET_DST_BUFFER: - case PUT_SRC_BUFFER: - free(ptl_wr); - break; - case UNKNOWN_BUFFER: - default: - log_error(nnti_debug_level, "unknown buffer type(%llu).", ptl_mem_hdl->type); - break; - } - } - -cleanup: - trios_stop_timer("NNTI_ptl_wait", total_time); - log_debug(debug_level, "exit"); - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on any buffer in buf_list to complete. - * - * Wait for remote_op on any buffer in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the request queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_ptl_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status) -{ - int rc=PTL_OK; - NNTI_result_t nnti_rc=NNTI_OK; - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr=NULL; - -// const NNTI_buffer_t *reg_buf=NULL; - const NNTI_work_request_t *wait_wr=NULL; - - int elapsed_time=0; - int timeout_per_call; - ptl_event_t event; - int which_eq=0; - - log_level debug_level=nnti_debug_level; - - log_debug(debug_level, "enter"); - - assert(wr_list); - assert(wr_count > 0); -// if (buf_count > 1) { -// /* if there is more than 1 buffer in the list, none of them can be a REQUEST_BUFFER */ -// for (uint32_t i=0;itransport_private)->type != REQUEST_BUFFER); -// } -// } -// } - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_ptl_wait(wr_list[0], timeout, status); - *which=0; - goto cleanup; - } - - if (is_any_wr_complete(wr_list, wr_count, which) == TRUE) { - log_debug(debug_level, "work request already complete (which=%u, wr_list[%d]=%p)", *which, *which, wr_list[*which]); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "work request NOT complete (buf_list=%p)", wr_list); - - timeout_per_call = MIN_TIMEOUT; - - while (1) { - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - return NNTI_ECANCELED; - } - - log_debug(debug_level, "waiting on eq_h(%d)", transport_global_data.data_eq_h); - - memset(&event, 0, sizeof(ptl_event_t)); - log_debug(debug_level, "lock before poll"); - // nthread_lock(&nnti_ptl_lock); - rc = PtlEQPoll(&transport_global_data.data_eq_h, 1, timeout_per_call, &event, &which_eq); - // nthread_unlock(&nnti_ptl_lock); - log_debug(debug_level, "polling status is %s", ptl_err_str[rc]); - - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\ttype = %d", event.type); - log_debug(debug_level, "\tinitiator = (%llu, %llu)", (unsigned long long)event.initiator.nid, (unsigned long long)event.initiator.pid); - log_debug(debug_level, "\tuid = %d", event.uid); - log_debug(debug_level, "\tjid = %d", event.jid); - log_debug(debug_level, "\tpt_index = %d", event.pt_index); - log_debug(debug_level, "\tmatch_bits = %d", event.match_bits); - log_debug(debug_level, "\trlength = %llu", (unsigned long long)event.rlength); - log_debug(debug_level, "\tmlength = %llu", (unsigned long long)event.mlength); - log_debug(debug_level, "\toffset = %llu", (unsigned long long)event.offset); - log_debug(debug_level, "\tmd_handle = %d", event.md_handle); - log_debug(debug_level, "\tmd.start = %p", event.md.start); - log_debug(debug_level, "\tmd.length = %d", event.md.length); - log_debug(debug_level, "\tmd.max_size = %d", event.md.max_size); - log_debug(debug_level, "\tmd.threshold = %d", event.md.threshold); - log_debug(debug_level, "\tmd.user_ptr = %p", event.md.user_ptr); - - - /* case 1: success */ - if (rc == PTL_OK) { - nnti_rc = NNTI_OK; - } - /* case 2: success, but some events were dropped */ - else if (rc == PTL_EQ_DROPPED) { - log_warn(debug_level, "PtlEQPoll dropped some events"); - log_warn(debug_level, "PtlEQPoll succeeded, but at least one event was dropped"); - nnti_rc = NNTI_OK; - } - /* case 3: timed out */ - else if (rc == PTL_EQ_EMPTY) { - elapsed_time += timeout_per_call; - - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout > 0) && (elapsed_time >= timeout))) { - log_debug(debug_level, "PtlEQPoll timed out: %s", - ptl_err_str[rc]); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - /* continue if the timeout has not expired */ - /* log_debug(debug_level, "timedout... continuing"); */ - - - - continue; - } - /* case 4: failure */ - else { - log_error(debug_level, "PtlEQPoll failed (eq_handle[%d]==%d): %s", - which_eq, transport_global_data.data_eq_h, ptl_err_str[rc]); - nnti_rc = NNTI_EIO; - break; - } - - wait_wr=decode_event_wr(wr_list[0], &event); - process_event(wait_wr, &event); - - if (is_any_wr_complete(wr_list, wr_count, which) == TRUE) { - break; - } - } - } - - - create_status(wr_list[*which], nnti_rc, status); - - if (logging_debug(debug_level)) { - fprint_NNTI_status(logger_get_file(), "status", - "end of NNTI_ptl_wait", status); - } - - if (nnti_rc==NNTI_OK) { - ptl_wr=PTL_WORK_REQUEST(wr_list[*which]); - assert(ptl_wr); - ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); - assert(ptl_mem_hdl); -// ptl_wr=ptl_mem_hdl->wr_queue.front(); -// assert(ptl_wr); -// ptl_mem_hdl->wr_queue.pop_front(); - - switch (ptl_mem_hdl->type) { - case REQUEST_BUFFER: - case RECEIVE_BUFFER: -#if defined(USE_RDMA_TARGET_ACK) - case GET_SRC_BUFFER: - case PUT_DST_BUFFER: - case RDMA_TARGET_BUFFER: -#endif -// repost_recv_work_request(wr_list[*which]); - break; - case SEND_BUFFER: - case GET_DST_BUFFER: - case PUT_SRC_BUFFER: - free(ptl_wr); - break; - case UNKNOWN_BUFFER: - default: - log_error(nnti_debug_level, "unknown buffer type(%llu).", ptl_mem_hdl->type); - break; - } - } - -cleanup: - log_debug(debug_level, "exit"); - return(nnti_rc); -} - -/** - * @brief Wait for remote_op on all buffers in buf_list to complete. - * - * Wait for remote_op on all buffers in buf_list to complete or timeout - * waiting. This is typically used to wait for a result or a bulk data - * transfer. The timeout is specified in milliseconds. A timeout of -1 - * means wait forever. A timeout of 0 means do not wait. - * - * Caveats: - * 1) All buffers in buf_list must be registered with the same transport. - * 2) You can't wait on the receive queue and RDMA buffers in the same call. Will probably be fixed in the future. - */ -NNTI_result_t NNTI_ptl_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status) -{ - int rc=PTL_OK; - NNTI_result_t nnti_rc=NNTI_OK; - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr=NULL; - - const NNTI_work_request_t *wait_wr=NULL; - - int elapsed_time=0; - int timeout_per_call; - ptl_event_t event; - int which_eq=0; - - log_level debug_level=nnti_debug_level; - - log_debug(debug_level, "enter"); - - assert(wr_list); - assert(wr_count > 0); -// if (buf_count > 1) { -// /* if there is more than 1 buffer in the list, none of them can be a REQUEST_BUFFER */ -// for (uint32_t i=0;itransport_private)->type != REQUEST_BUFFER); -// } -// } -// } - assert(status); - - if (wr_count == 1) { - nnti_rc=NNTI_ptl_wait(wr_list[0], timeout, status[0]); - goto cleanup; - } - - if (is_all_wr_complete(wr_list, wr_count) == TRUE) { - log_debug(debug_level, "all buffer ops already complete (buf_list=%p)", wr_list); - nnti_rc = NNTI_OK; - } else { - log_debug(debug_level, "all buffer ops NOT complete (buf_list=%p)", wr_list); - - timeout_per_call = MIN_TIMEOUT; - - while (1) { - if (trios_exit_now()) { - log_debug(debug_level, "caught abort signal"); - return NNTI_ECANCELED; - } - - log_debug(debug_level, "waiting on eq_h(%d)", transport_global_data.data_eq_h); - - memset(&event, 0, sizeof(ptl_event_t)); - log_debug(debug_level, "lock before poll"); - // nthread_lock(&nnti_ptl_lock); - rc = PtlEQPoll(&transport_global_data.data_eq_h, 1, timeout_per_call, &event, &which_eq); - // nthread_unlock(&nnti_ptl_lock); - log_debug(debug_level, "polling status is %s", ptl_err_str[rc]); - - log_debug(debug_level, "Poll Event= {"); - log_debug(debug_level, "\ttype = %d", event.type); - log_debug(debug_level, "\tinitiator = (%llu, %llu)", (unsigned long long)event.initiator.nid, (unsigned long long)event.initiator.pid); - log_debug(debug_level, "\tuid = %d", event.uid); - log_debug(debug_level, "\tjid = %d", event.jid); - log_debug(debug_level, "\tpt_index = %d", event.pt_index); - log_debug(debug_level, "\tmatch_bits = %d", event.match_bits); - log_debug(debug_level, "\trlength = %llu", (unsigned long long)event.rlength); - log_debug(debug_level, "\tmlength = %llu", (unsigned long long)event.mlength); - log_debug(debug_level, "\toffset = %llu", (unsigned long long)event.offset); - log_debug(debug_level, "\tmd_handle = %d", event.md_handle); - log_debug(debug_level, "\tmd.start = %p", event.md.start); - log_debug(debug_level, "\tmd.length = %d", event.md.length); - log_debug(debug_level, "\tmd.max_size = %d", event.md.max_size); - log_debug(debug_level, "\tmd.threshold = %d", event.md.threshold); - log_debug(debug_level, "\tmd.user_ptr = %p", event.md.user_ptr); - - - /* case 1: success */ - if (rc == PTL_OK) { - nnti_rc = NNTI_OK; - } - /* case 2: success, but some events were dropped */ - else if (rc == PTL_EQ_DROPPED) { - log_warn(debug_level, "PtlEQPoll dropped some events"); - log_warn(debug_level, "PtlEQPoll succeeded, but at least one event was dropped"); - nnti_rc = NNTI_OK; - } - /* case 3: timed out */ - else if (rc == PTL_EQ_EMPTY) { - elapsed_time += timeout_per_call; - - /* if the caller asked for a legitimate timeout, we need to exit */ - if (((timeout > 0) && (elapsed_time >= timeout))) { - log_debug(debug_level, "PtlEQPoll timed out: %s", - ptl_err_str[rc]); - nnti_rc = NNTI_ETIMEDOUT; - break; - } - /* continue if the timeout has not expired */ - /* log_debug(debug_level, "timedout... continuing"); */ - - - - continue; - } - /* case 4: failure */ - else { - log_error(debug_level, "PtlEQPoll failed (eq_handle[%d]==%d): %s", - which_eq, transport_global_data.data_eq_h, ptl_err_str[rc]); - nnti_rc = NNTI_EIO; - break; - } - - wait_wr=decode_event_wr(wr_list[0], &event); - process_event(wait_wr, &event); - - if (is_all_wr_complete(wr_list, wr_count) == TRUE) { - break; - } - } - } - - - for (uint32_t i=0;ireg_buf); - assert(ptl_mem_hdl); -// ptl_wr=ptl_mem_hdl->wr_queue.front(); -// assert(ptl_wr); -// ptl_mem_hdl->wr_queue.pop_front(); - - if (nnti_rc==NNTI_OK) { - switch (ptl_mem_hdl->type) { - case REQUEST_BUFFER: - case RECEIVE_BUFFER: -#if defined(USE_RDMA_TARGET_ACK) - case GET_SRC_BUFFER: - case PUT_DST_BUFFER: - case RDMA_TARGET_BUFFER: -#endif -// repost_recv_work_request(wr_list[i]); - break; - case SEND_BUFFER: - case GET_DST_BUFFER: - case PUT_SRC_BUFFER: - free(ptl_wr); - break; - case UNKNOWN_BUFFER: - default: - log_error(nnti_debug_level, "unknown buffer type(%llu).", ptl_mem_hdl->type); - break; - } - } - } - -cleanup: - log_debug(debug_level, "exit"); - return(nnti_rc); -} - -/** - * @brief Disable this transport. - * - * Shutdown the transport. Any outstanding sends, gets and puts will be - * canceled. Any new transport requests will fail. - * - */ -NNTI_result_t NNTI_ptl_fini ( - const NNTI_transport_t *trans_hdl) -{ -// PtlFini(); - nthread_lock_fini(&nnti_ptl_lock); - - ptl_initialized=false; - -#if !defined(HAVE_TRIOS_CRAYPORTALS) && defined(HAVE_TRIOS_MPI) - if (transport_global_data.init_called_mpi_init) { - MPI_Finalize(); - } -#endif - - return(NNTI_OK); -} - - - - -static void set_req_pid(NNTI_pid *pid) -{ - log_debug(nnti_debug_level, "enter (pid=%d)", *pid); - - transport_global_data.init_called_mpi_init=false; - - *pid=PTL_PID_ANY; - -#if !defined(HAVE_TRIOS_CRAYPORTALS) && defined(HAVE_TRIOS_MPI) - // Schutt's Portals doesn't properly assign a pid when you pass PTL_PID_ANY. Fix it here. - int initialized=0; - int rank; - MPI_Initialized(&initialized); - if (!initialized) { - MPI_Init(NULL, NULL); - transport_global_data.init_called_mpi_init=true; - } - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(nnti_debug_level, "rank=%d", rank); - - *pid = 128 + rank; -#endif - - log_debug(nnti_debug_level, "exit (pid=%d)", *pid); -} - - -//static portals_work_request *decode_work_request( -// const ptl_event_t *event) -//{ -// log_level debug_level = nnti_debug_level; -// -// const NNTI_buffer_t *event_buf=NULL; -// portals_memory_handle *ptl_mem_hdl=NULL; -// NNTI_work_request_t *wr=NULL; -// portals_work_request *ptl_wr=NULL; -// portals_work_request *debug_wr=NULL; -// -// log_debug(debug_level, "enter"); -// -// event_buf=(NNTI_buffer_t *)event->md.user_ptr; -// assert(event_buf); -// ptl_mem_hdl=PTL_MEM_HDL(event_buf); -// assert(ptl_mem_hdl); -// -// wr_queue_iter_t i; -// for (i=ptl_mem_hdl->wr_queue.begin(); i != ptl_mem_hdl->wr_queue.end(); i++) { -// assert(*i); -// ptl_wr=PTL_WORK_REQUEST(*i); -// if (is_wr_complete(ptl_wr) == FALSE) { -// // work request is incomplete, check if it matches this event -// switch(ptl_mem_hdl->type) { -// case REQUEST_BUFFER: -// if (((ptl_wr)->src_offset == event->offset) && -// ((ptl_wr)->length == event->mlength)) { -// -// wr=*i; -// } else { -// log_debug(debug_level, "work request doesn't match (wr=%p)", ptl_wr); -// } -// break; -// case SEND_BUFFER: -// case PUT_SRC_BUFFER: -// if (((ptl_wr)->src_offset == event->offset) && -// ((ptl_wr)->length == event->mlength)) { -// -// wr=*i; -// } else { -// log_debug(debug_level, "work request doesn't match (wr=%p)", ptl_wr); -// } -// break; -// case GET_DST_BUFFER: -// if (((ptl_wr)->dst_offset == event->offset) && -// ((ptl_wr)->length == event->mlength)) { -// -// wr=*i; -// } else { -// log_debug(debug_level, "work request doesn't match (wr=%p)", ptl_wr); -// } -// break; -// case RECEIVE_BUFFER: -// case GET_SRC_BUFFER: -// case PUT_DST_BUFFER: -// case RDMA_TARGET_BUFFER: -// wr=*i; -// break; -// default: -// log_debug(debug_level, "unknown event type %d (event_buf==%p)", ptl_mem_hdl->type, event_buf); -// break; -// } -// if (wr) { -// break; -// } -// } else { -// log_debug(debug_level, "work request is already complete (wr=%p)", ptl_wr); -// } -// } -// -// if (!wr) { -// for (i=ptl_mem_hdl->wr_queue.begin(); i != ptl_mem_hdl->wr_queue.end(); i++) { -// debug_wr=PTL_WORK_REQUEST(*i); -// log_debug(LOG_ALL, "e.offset=%llu, e.rlength=%llu, e.mlength=%llu, wr=%p, wr.length=%llu, wr.src_offset=%llu, wr.dst_offset=%llu, wr.is_complete=%d", -// (uint64_t)event->offset, (uint64_t)event->rlength, (uint64_t)event->mlength, -// debug_wr, -// (uint64_t)debug_wr->length, (uint64_t)debug_wr->src_offset, (uint64_t)debug_wr->dst_offset, -// (is_wr_complete(debug_wr)==TRUE)); -// } -// } -// assert(wr); -// -// log_debug(debug_level, "exit (wr==%p)", wr); -// -// return(PTL_WORK_REQUEST(wr)); -//} - -static const NNTI_work_request_t *decode_event_wr( - const NNTI_work_request_t *wait_wr, - const ptl_event_t *event) -{ - const NNTI_buffer_t *event_buf=NULL; - const NNTI_work_request_t *event_wr=NULL; - - const NNTI_buffer_t *wait_buf=NULL; - portals_work_request *ptl_wr=NULL; - - portals_memory_handle *ptl_mem_hdl=NULL; - - log_debug(nnti_debug_level, "enter"); - - assert(wait_wr); - - ptl_wr=PTL_WORK_REQUEST(wait_wr); - wait_buf=ptl_wr->reg_buf; - - switch (event->type) { - /* these are the events for a PUT initiator */ - case PTL_EVENT_SEND_START: - log_debug(nnti_debug_level, "got PTL_EVENT_SEND_START - event->user_ptr is a work request"); - event_wr=(NNTI_work_request_t *)event->md.user_ptr; - break; - case PTL_EVENT_SEND_END: - log_debug(nnti_debug_level, "got PTL_EVENT_SEND_END - event->user_ptr is a work request"); - event_wr=(NNTI_work_request_t *)event->md.user_ptr; - break; - case PTL_EVENT_ACK: - log_debug(nnti_debug_level, "got PTL_EVENT_ACK - event->user_ptr is a work request"); - event_wr=(NNTI_work_request_t *)event->md.user_ptr; - break; - case PTL_EVENT_UNLINK: - log_debug(nnti_debug_level, "got PTL_EVENT_UNLINK - event->user_ptr is a work request"); - event_wr=(NNTI_work_request_t *)event->md.user_ptr; - break; - - - /* these are the events for a GET initiator */ - case PTL_EVENT_REPLY_START: - log_debug(nnti_debug_level,"got PTL_EVENT_REPLY_START - event->user_ptr is a work request"); - event_wr=(NNTI_work_request_t *)event->md.user_ptr; - break; - case PTL_EVENT_REPLY_END: - log_debug(nnti_debug_level,"got PTL_EVENT_REPLY_END - event->user_ptr is a work request"); - event_wr=(NNTI_work_request_t *)event->md.user_ptr; - break; - - - /* these are the events for a PUT target */ - case PTL_EVENT_PUT_START: - log_debug(nnti_debug_level, "got PTL_EVENT_PUT_START - "); - if (event->pt_index == NNTI_REQ_PT_INDEX) { - event_wr=wait_wr; - log_debug(nnti_debug_level, "the wait work request is a REQUEST BUFFER wr, so event.md.user_ptr is the index of the request buffer."); - } else { - event_buf=(NNTI_buffer_t *)event->md.user_ptr; - ptl_mem_hdl=PTL_MEM_HDL(event_buf); - assert(ptl_mem_hdl); - event_wr=ptl_mem_hdl->wr_queue.front(); - } - break; - case PTL_EVENT_PUT_END: - log_debug(nnti_debug_level, "got PTL_EVENT_PUT_END - "); - if (event->pt_index == NNTI_REQ_PT_INDEX) { - event_wr=wait_wr; - log_debug(nnti_debug_level, "the wait work request is a REQUEST BUFFER wr, so event.md.user_ptr is the index of the request buffer."); - } else { - event_buf=(NNTI_buffer_t *)event->md.user_ptr; - ptl_mem_hdl=PTL_MEM_HDL(event_buf); - assert(ptl_mem_hdl); - event_wr=ptl_mem_hdl->wr_queue.front(); - } - break; - - - case PTL_EVENT_GET_START: - log_debug(nnti_debug_level, "got PTL_EVENT_GET_START - GET targets should NOT generate events"); - break; - case PTL_EVENT_GET_END: - log_debug(nnti_debug_level, "got PTL_EVENT_GET_END - GET targets should NOT generate events"); - break; - - - default: - log_error(nnti_debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); -// rc = NNTI_EINVAL; - - } - -// if ((ptl_wr != NULL) && -// (PTL_MEM_HDL(ptl_wr->reg_buf)->type == REQUEST_BUFFER)) { -// /* if the buffer is a request queue, then the wait_wr is the event_wr */ -// event_wr=wait_wr; -// log_debug(nnti_debug_level, "the wait work request is a REQUEST BUFFER wr, so event.md.user_ptr is the index of the request buffer."); -// } else if ((event->initiator.nid==transport_global_data.me.nid) && (event->initiator.pid==transport_global_data.me.pid)) { -// /* if I am the initiator, then the event user_ptr is a work request pointer */ -// -// log_debug(nnti_debug_level, "I am the initiator, so the event user_ptr is a work request pointer"); -// -// event_wr=(NNTI_work_request_t *)event->md.user_ptr; -// ptl_mem_hdl=PTL_MEM_HDL(PTL_WORK_REQUEST(event_wr)->reg_buf); -// assert(ptl_mem_hdl); -// -// if (event_wr == wait_wr) { -// log_debug(nnti_debug_level, "the wc matches the wait work request (eq=%d, user_ptr=%p, wait_wr=%p)", -// ptl_mem_hdl->eq_h, (void *)event->md.user_ptr, wait_wr); -// } else { -// log_debug(nnti_debug_level, "the wc does NOT match the wait buffer (eq=%d, user_ptr=%p, wait_wr=%p)", -// ptl_mem_hdl->eq_h, (void *)event->md.user_ptr, wait_wr); -// } -// } else { -// /* if I am NOT the initiator, then the event user_ptr is a pointer to the target buffer */ -// event_buf=(NNTI_buffer_t *)event->md.user_ptr; -// ptl_mem_hdl=PTL_MEM_HDL(event_buf); -// assert(ptl_mem_hdl); -// -// if (event_buf == PTL_WORK_REQUEST(wait_wr)->reg_buf) { -// log_debug(nnti_debug_level, "the wc matches the wait buffer (eq=%d, user_ptr=%p, wait_buf=%p)", -// ptl_mem_hdl->eq_h, (void *)event->md.user_ptr, wait_buf); -// } else { -// log_debug(nnti_debug_level, "the wc does NOT match the wait buffer (eq=%d, user_ptr=%p, wait_buf=%p)", -// ptl_mem_hdl->eq_h, (void *)event->md.user_ptr, wait_buf); -// } -// } - - log_debug(nnti_debug_level, "exit (event_wr==%p)", event_wr); - - return(event_wr); -} - - -static int process_event( - const NNTI_work_request_t *wr, - const ptl_event_t *event) -{ - int rc=NNTI_OK; - - portals_memory_handle *ptl_mem_hdl=NULL; - portals_work_request *ptl_wr =NULL; - - log_level debug_level = nnti_debug_level; - - ptl_wr=PTL_WORK_REQUEST(wr); - assert(ptl_wr); - ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); - assert(ptl_mem_hdl); - -// if (ptl_mem_hdl->type != REQUEST_BUFFER) { -// ptl_wr=decode_work_request(event); -// } else { -// ptl_wr=ptl_mem_hdl->wr_queue.front(); -// } - if (ptl_mem_hdl->type == REQUEST_BUFFER) { - NNTI_work_request_t *tmp_wr=NULL; - tmp_wr=ptl_mem_hdl->wr_queue.front(); - assert(tmp_wr); - ptl_wr=PTL_WORK_REQUEST(tmp_wr); - } - assert(ptl_wr); - - ptl_wr->last_event=*event; - - log_debug(debug_level, "wr=%p; ptl_wr=%p; ptl_wr->last_op=%d", wr, ptl_wr, ptl_wr->last_op); - switch (ptl_mem_hdl->type) { - case SEND_BUFFER: - case PUT_SRC_BUFFER: - switch (event->type) { - case PTL_EVENT_SEND_START: - log_debug(debug_level, "got PTL_EVENT_SEND_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_initiator.send_start = TRUE; - break; - case PTL_EVENT_SEND_END: - log_debug(debug_level, "got PTL_EVENT_SEND_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_initiator.send_end = TRUE; - break; - case PTL_EVENT_ACK: - log_debug(debug_level, "got PTL_EVENT_ACK - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_initiator.ack = TRUE; - break; - case PTL_EVENT_UNLINK: - log_debug(debug_level, "got PTL_EVENT_UNLINK - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_initiator.unlink = TRUE; - break; - default: - log_error(debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - rc = NNTI_EINVAL; - goto cleanup; - } - break; - case GET_DST_BUFFER: - switch (event->type) { - case PTL_EVENT_SEND_START: - log_debug(debug_level, "got PTL_EVENT_SEND_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_initiator.send_start = TRUE; - break; - case PTL_EVENT_SEND_END: - log_debug(debug_level, "got PTL_EVENT_SEND_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_initiator.send_end = TRUE; - break; - case PTL_EVENT_REPLY_START: - log_debug(debug_level,"got PTL_EVENT_REPLY_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_initiator.reply_start = TRUE; - break; - case PTL_EVENT_REPLY_END: - log_debug(debug_level,"got PTL_EVENT_REPLY_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_initiator.reply_end = TRUE; - break; - case PTL_EVENT_UNLINK: - log_debug(debug_level, "got PTL_EVENT_UNLINK - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_initiator.unlink = TRUE; - break; - default: - log_error(debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - rc = NNTI_EINVAL; - goto cleanup; - } - break; - case REQUEST_BUFFER: - case RECEIVE_BUFFER: - switch (event->type) { - case PTL_EVENT_PUT_START: - log_debug(debug_level, "got PTL_EVENT_PUT_START - new request - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - - break; - case PTL_EVENT_PUT_END: - log_debug(debug_level, "got PTL_EVENT_PUT_END - new request - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_target.put_start = TRUE; - ptl_wr->op_state.put_target.put_end = TRUE; - break; - default: - log_error(debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - rc = NNTI_EINVAL; - goto cleanup; - } - break; - case PUT_DST_BUFFER: - switch (event->type) { - case PTL_EVENT_PUT_START: - log_debug(debug_level, "got PTL_EVENT_PUT_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_target.put_start = TRUE; - break; - case PTL_EVENT_PUT_END: - log_debug(debug_level, "got PTL_EVENT_PUT_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_target.put_end = TRUE; - break; - default: - log_error(debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - rc = NNTI_EINVAL; - goto cleanup; - } - break; - case GET_SRC_BUFFER: - switch (event->type) { - case PTL_EVENT_GET_START: - log_debug(debug_level, "got PTL_EVENT_GET_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_target.get_start = TRUE; - break; - case PTL_EVENT_GET_END: - log_debug(debug_level, "got PTL_EVENT_GET_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_target.get_end = TRUE; - break; - default: - log_error(debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - rc = NNTI_EINVAL; - goto cleanup; - } - break; - case RDMA_TARGET_BUFFER: - switch (event->type) { - case PTL_EVENT_PUT_START: - log_debug(debug_level, "got PTL_EVENT_PUT_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_target.put_start = TRUE; - break; - case PTL_EVENT_PUT_END: - log_debug(debug_level, "got PTL_EVENT_PUT_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.put_target.put_end = TRUE; - break; - case PTL_EVENT_GET_START: - log_debug(debug_level, "got PTL_EVENT_GET_START - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_target.get_start = TRUE; - break; - case PTL_EVENT_GET_END: - log_debug(debug_level, "got PTL_EVENT_GET_END - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - ptl_mem_hdl->eq_h, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - ptl_wr->op_state.get_target.get_end = TRUE; - break; - default: - log_error(debug_level, "unrecognized event type: %d - event arrived on eq %d - initiator = (%4llu, %4llu, %4d)", - event->type, (unsigned long long)event->initiator.nid,(unsigned long long)event->initiator.pid, event->link); - rc = NNTI_EINVAL; - goto cleanup; - } - break; - case UNKNOWN_BUFFER: - default: - break; - } - - if (event->ni_fail_type != PTL_NI_OK) { - log_error(debug_level, "failed on put end: ni_fail_type=%d\n", - event->ni_fail_type); - rc = event->ni_fail_type; - } - -cleanup: - return (rc); -} - -//static NNTI_result_t post_recv_work_request( -// NNTI_buffer_t *reg_buf) -//{ -// portals_work_request *ptl_wr=NULL; -// portals_memory_handle *ptl_mem_hdl=NULL; -// -// log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); -// -// ptl_mem_hdl=PTL_MEM_HDL(reg_buf); -// assert(ptl_mem_hdl); -// -// ptl_wr=(portals_work_request *)calloc(1, sizeof(portals_work_request)); -// assert(ptl_wr); -// ptl_wr->reg_buf = reg_buf; -// -// memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); -// -// ptl_mem_hdl->wr_queue.push_back(ptl_wr); -// -// log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); -// -// return(NNTI_OK); -//} - -//static NNTI_result_t repost_recv_work_request( -// NNTI_work_request_t *wr) -//{ -// portals_work_request *ptl_wr=NULL; -// portals_memory_handle *ptl_mem_hdl=NULL; -// -// log_debug(nnti_debug_level, "enter (wr=%p)", wr); -// -// ptl_wr=PTL_WORK_REQUEST(wr); -// assert(ptl_wr); -// ptl_mem_hdl=PTL_MEM_HDL(ptl_wr->reg_buf); -// assert(ptl_mem_hdl); -// -// memset(&ptl_wr->op_state, 0, sizeof(ptl_op_state_t)); -// -// ptl_mem_hdl->wr_queue.push_back(ptl_wr); -// -// log_debug(nnti_debug_level, "exit (wr=%p)", wr); -// -// return(NNTI_OK); -//} - -static int8_t is_wr_complete( - portals_work_request *wr) -{ - int rc=FALSE; - portals_memory_handle *ptl_mem_hdl=NULL; -// log_level debug_level = nnti_debug_level; - - log_debug(nnti_debug_level, "enter (wr=%p)", wr); - - ptl_mem_hdl=PTL_MEM_HDL(wr->reg_buf); - assert(ptl_mem_hdl); - - switch (ptl_mem_hdl->type) { - case SEND_BUFFER: - case PUT_SRC_BUFFER: - if ((wr->op_state.put_initiator.send_start==TRUE) && - (wr->op_state.put_initiator.send_end==TRUE) && - (wr->op_state.put_initiator.ack==TRUE) && - (wr->op_state.put_initiator.unlink==TRUE)) { - wr->last_op=PTL_OP_PUT_INITIATOR; - rc = TRUE; - } - case GET_DST_BUFFER: - /* cray portals */ - if ((wr->op_state.get_initiator.send_start==TRUE) && - (wr->op_state.get_initiator.send_end==TRUE) && - (wr->op_state.get_initiator.reply_start==TRUE) && - (wr->op_state.get_initiator.reply_end==TRUE) && - (wr->op_state.get_initiator.unlink==TRUE)) { - wr->last_op=PTL_OP_GET_INITIATOR; - rc = TRUE; - break; - } - /* schutt portals */ - if ((wr->op_state.get_initiator.reply_start==TRUE) && - (wr->op_state.get_initiator.reply_end==TRUE) && - (wr->op_state.get_initiator.unlink==TRUE)) { - wr->last_op=PTL_OP_GET_INITIATOR; - rc = TRUE; - break; - } - break; - case PUT_DST_BUFFER: - if ((wr->op_state.put_target.put_start==TRUE) && - (wr->op_state.put_target.put_end==TRUE)) { - wr->last_op=PTL_OP_PUT_TARGET; - rc = TRUE; - } - break; - case GET_SRC_BUFFER: - if ((wr->op_state.get_target.get_start==TRUE) && - (wr->op_state.get_target.get_end==TRUE)) { - wr->last_op=PTL_OP_GET_TARGET; - rc = TRUE; - } - break; - case REQUEST_BUFFER: - if ((wr->op_state.put_target.put_start==TRUE) && - (wr->op_state.put_target.put_end==TRUE)) { - wr->last_op=PTL_OP_NEW_REQUEST; - rc = TRUE; - } - break; - case RECEIVE_BUFFER: - if ((wr->op_state.put_target.put_start==TRUE) && - (wr->op_state.put_target.put_end==TRUE)) { - wr->last_op=PTL_OP_RECEIVE; - rc = TRUE; - } - break; - case RDMA_TARGET_BUFFER: - if ((wr->op_state.get_target.get_start==TRUE) && - (wr->op_state.get_target.get_end==TRUE)) { - wr->last_op=PTL_OP_GET_TARGET; - rc = TRUE; - } - if ((wr->op_state.put_target.put_start==TRUE) && - (wr->op_state.put_target.put_end==TRUE)) { - wr->last_op=PTL_OP_PUT_TARGET; - rc = TRUE; - } - break; - case UNKNOWN_BUFFER: - default: - break; - } - - log_debug(nnti_debug_level, "exit (rc=%d)", rc); - - return(rc); -} - -static int8_t is_any_wr_complete( - portals_work_request **wr_list, - const uint32_t wr_count, - uint32_t *which) -{ - int8_t rc=FALSE; - - log_debug(nnti_debug_level, "enter"); - - for (uint32_t i=0;iwr_queue.empty()) { -// log_debug(nnti_debug_level, "work request queue is empty"); -// } else { -// wr_queue_iter_t i; -// for (i=ptl_mem_hdl->wr_queue.begin(); i != ptl_mem_hdl->wr_queue.end(); i++) { -// wr=*i; -// assert(wr); -// if (is_wr_complete(wr) == FALSE) { -// break; -// } -// } -// } -// -// log_debug(nnti_debug_level, "exit (wr=%p)", wr); -// return(PTL_WORK_REQUEST(wr)); -//} - -//static int8_t is_wr_queue_empty( -// const NNTI_buffer_t *reg_buf) -//{ -// int8_t rc=FALSE; -// portals_memory_handle *ptl_mem_hdl=NULL; -// -// log_debug(nnti_debug_level, "enter"); -// -// ptl_mem_hdl=PTL_MEM_HDL(reg_buf); -// assert(ptl_mem_hdl); -// -// if (ptl_mem_hdl->wr_queue.empty()) { -// rc=TRUE; -// } -// -// log_debug(nnti_debug_level, "exit (rc=%d)", rc); -// return(rc); -//} -// -// -//static int8_t is_buf_op_complete( -// const NNTI_buffer_t *reg_buf) -//{ -// int8_t rc=FALSE; -// portals_memory_handle *ptl_mem_hdl=NULL; -// NNTI_work_request_t *wr =NULL; -// portals_work_request *ptl_wr=NULL; -//// log_level debug_level = nnti_debug_level; -// -// log_debug(nnti_debug_level, "enter (reg_buf=%p)", reg_buf); -// -// ptl_mem_hdl=(portals_memory_handle *)reg_buf->transport_private; -// assert(ptl_mem_hdl); -// -// if (is_wr_queue_empty(reg_buf) == TRUE) { -// log_debug(nnti_debug_level, "work request queue is empty - return FALSE"); -// rc=FALSE; -// } else { -// wr=ptl_mem_hdl->wr_queue.front(); -// assert(wr); -// -// rc = is_wr_complete(wr); -// } -// -// if (rc==TRUE) { -// log_debug(nnti_debug_level, "op is complete"); -// } -// log_debug(nnti_debug_level, "exit (reg_buf=%p)", reg_buf); -// -// return(rc); -//} -// -//static int8_t is_any_buf_op_complete( -// const NNTI_buffer_t **buf_list, -// const uint32_t buf_count, -// uint32_t *which) -//{ -// int8_t rc=FALSE; -// -// log_debug(nnti_debug_level, "enter"); -// -// for (uint32_t i=0;iop = wr->ops; - status->result = (NNTI_result_t)nnti_rc; - if (nnti_rc==NNTI_OK) { - ptl_wr=(portals_work_request *)wr->transport_private; - assert(ptl_wr); - ptl_mem_hdl=(portals_memory_handle *)ptl_wr->reg_buf->transport_private; - assert(ptl_mem_hdl); - - status->start = (uint64_t)ptl_wr->last_event.md.start; - status->offset = ptl_wr->last_event.offset; - status->length = ptl_wr->last_event.mlength; - switch (ptl_wr->last_op) { - case PTL_OP_PUT_INITIATOR: - case PTL_OP_GET_TARGET: - case PTL_OP_SEND: - create_peer(&status->src, transport_global_data.me.nid, transport_global_data.me.pid); // allocates url - create_peer(&status->dest, ptl_wr->last_event.initiator.nid, ptl_wr->last_event.initiator.pid); // allocates url - break; - case PTL_OP_GET_INITIATOR: - case PTL_OP_PUT_TARGET: - case PTL_OP_NEW_REQUEST: - case PTL_OP_RECEIVE: - create_peer(&status->src, ptl_wr->last_event.initiator.nid, ptl_wr->last_event.initiator.pid); // allocates url - create_peer(&status->dest, transport_global_data.me.nid, transport_global_data.me.pid); // allocates url - break; - } - } -} - -static void create_peer(NNTI_peer_t *peer, ptl_nid_t nid, ptl_pid_t pid) -{ - log_debug(nnti_debug_level, "enter"); - - sprintf(peer->url, "ptl://%u:%u/", nid, pid); - - peer->peer.transport_id = NNTI_TRANSPORT_PORTALS; - peer->peer.NNTI_remote_process_t_u.portals.nid = nid; - peer->peer.NNTI_remote_process_t_u.portals.pid = pid; - - log_debug(nnti_debug_level, "exit"); -} diff --git a/packages/trios/libraries/nessie/nnti/nnti_ptls.h b/packages/trios/libraries/nessie/nnti/nnti_ptls.h deleted file mode 100644 index 04583456a989..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_ptls.h +++ /dev/null @@ -1,313 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * nnti_ptls.h - * - * Created on: Jan 13, 2011 - * Author: thkorde - */ - -#ifndef NNTI_PTLS_H_ -#define NNTI_PTLS_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "Trios_config.h" - -#include "Trios_nnti.h" -#include "nnti_internal.h" - - - -/* Figure out which include files we need for Portals */ -#if defined(HAVE_TRIOS_PORTALS) -#include -#elif defined(HAVE_TRIOS_CRAYPORTALS) -#include -#else -#error There is no portals3.h file -#endif - -#if defined(HAVE_TRIOS_P3NAL_UTCP_H) -#include -#endif - -#if defined(HAVE_TRIOS_P3RT_P3RT_H) -#include -#endif - -/* Fix some other inconsistencies across Portals versions */ - -#ifndef PTL_IFACE_SERVER -#define PTL_IFACE_SERVER PTL_IFACE_DEFAULT -#endif - -#ifndef PTL_IFACE_CLIENT -#define PTL_IFACE_CLIENT PTL_IFACE_DEFAULT -#endif - - -#ifndef HAVE_TRIOS_PTL_NO_ACK_REQ -#ifdef HAVE_TRIOS_PTL_NOACK_REQ -#define PTL_NO_ACK_REQ PTL_NOACK_REQ -#else -#error PTL_NO_ACK_REQ is not defined -#endif -#endif - -/* Cray extensions */ -#ifndef PTL_IFACE_DUP -#define PTL_IFACE_DUP PTL_OK -#endif - -#ifndef PTL_MD_EVENT_AUTO_UNLINK_ENABLE -#define PTL_MD_EVENT_AUTO_UNLINK_ENABLE 0 -#endif - -#ifndef PTL_MD_EVENT_MANUAL_UNLINK_ENABLE -#define PTL_MD_EVENT_MANUAL_UNLINK_ENABLE 0 -#endif - -#ifndef HAVE_TRIOS_PTLERRORSTR -#define PtlErrorStr(a) "" -#endif - -#ifndef HAVE_TRIOS_PTLNIFAILSTR -#define PtlNIFailStr(a,b) "" -#endif - -#ifndef HAVE_TRIOS_PTLEVENTKINDSTR -#define PtlEventKindStr(a) "" -#endif - -#ifndef HAVE_TRIOS_PTL_TIME_T -typedef uint32_t ptl_time_t; -#endif - -#ifndef HAVE_TRIOS_PTL_EQ_HANDLER_T -typedef void (*ptl_eq_handler_t)(ptl_event_t *event); -#endif - -#ifndef PTL_EQ_HANDLER_NONE -#define PTL_EQ_HANDLER_NONE (ptl_eq_handler_t)NULL -#endif - - - - - - - - - - - -#define PORTALS_SET_MATCH_ANY(p) \ -(p)->peer.transport_id = NNTI_TRANSPORT_PORTALS; \ -(p)->peer.NNTI_remote_process_t_u.portals.nid = PTL_NID_ANY; \ -(p)->peer.NNTI_remote_process_t_u.portals.pid = PTL_PID_ANY; - - - -NNTI_result_t NNTI_ptl_init ( - const NNTI_transport_id_t trans_id, - const char *my_url, - NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_ptl_get_url ( - const NNTI_transport_t *trans_hdl, - char *url, - const uint64_t maxlen); - -NNTI_result_t NNTI_ptl_connect ( - const NNTI_transport_t *trans_hdl, - const char *url, - const int timeout, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_ptl_disconnect ( - const NNTI_transport_t *trans_hdl, - NNTI_peer_t *peer_hdl); - -NNTI_result_t NNTI_ptl_alloc ( - const NNTI_transport_t *trans_hdl, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ptl_free ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ptl_register_memory ( - const NNTI_transport_t *trans_hdl, - char *buffer, - const uint64_t element_size, - const uint64_t num_elements, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ptl_register_segments ( - const NNTI_transport_t *trans_hdl, - char **segments, - const uint64_t *segment_lengths, - const uint64_t num_segments, - const NNTI_buf_ops_t ops, - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ptl_unregister_memory ( - NNTI_buffer_t *reg_buf); - -NNTI_result_t NNTI_ptl_send ( - const NNTI_peer_t *peer_hdl, - const NNTI_buffer_t *msg_hdl, - const NNTI_buffer_t *dest_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_put ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_get ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_offset, - const uint64_t src_length, - const NNTI_buffer_t *dest_buffer_hdl, - const uint64_t dest_offset, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_scatter ( - const NNTI_buffer_t *src_buffer_hdl, - const uint64_t src_length, - const NNTI_buffer_t **dest_buffer_list, - const uint64_t dest_count, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_gather ( - const NNTI_buffer_t **src_buffer_list, - const uint64_t src_length, - const uint64_t src_count, - const NNTI_buffer_t *dest_buffer_hdl, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_atomic_set_callback ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - NNTI_callback_fn_t cbfunc, - void *context); - -NNTI_result_t NNTI_ptl_atomic_read ( - const NNTI_transport_t *trans_hdl, - const uint64_t local_atomic, - int64_t *value); - -NNTI_result_t NNTI_ptl_atomic_fop ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t operand, - const NNTI_atomic_op_t op, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_atomic_cswap ( - const NNTI_transport_t *trans_hdl, - const NNTI_peer_t *peer_hdl, - const uint64_t target_atomic, - const uint64_t result_atomic, - const int64_t compare_operand, - const int64_t swap_operand, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_create_work_request ( - NNTI_buffer_t *reg_buf, - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_clear_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_destroy_work_request ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_cancel ( - NNTI_work_request_t *wr); - -NNTI_result_t NNTI_ptl_cancelall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count); - -NNTI_result_t NNTI_ptl_interrupt ( - const NNTI_transport_t *trans_hdl); - -NNTI_result_t NNTI_ptl_wait ( - NNTI_work_request_t *wr, - const int timeout, - NNTI_status_t *status); - -NNTI_result_t NNTI_ptl_waitany ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - uint32_t *which, - NNTI_status_t *status); - -NNTI_result_t NNTI_ptl_waitall ( - NNTI_work_request_t **wr_list, - const uint32_t wr_count, - const int timeout, - NNTI_status_t **status); - -NNTI_result_t NNTI_ptl_fini ( - const NNTI_transport_t *trans_hdl); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_PTLS_H_*/ diff --git a/packages/trios/libraries/nessie/nnti/nnti_utils.c b/packages/trios/libraries/nessie/nnti/nnti_utils.c deleted file mode 100644 index dc66958a01a4..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_utils.c +++ /dev/null @@ -1,219 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * nnti_utils.c - * - * Created on: Feb 16, 2011 - * Author: thkorde - */ - - -#include -#include -#include - -#include "nnti_utils.h" - -#include "nnti_internal.h" -#include "Trios_logger.h" - -NNTI_result_t nnti_url_get_transport(const char *url, char *outstr, const int maxlen) -{ - int translen=0; - char *sep=NULL; - - sep=strstr(url, "://"); - if (sep == NULL) { - /* invalid URL */ - return(NNTI_EINVAL); - } else { - translen = sep-url; - if (translen >= maxlen) { - /* */ - memcpy(outstr, url, maxlen-1); - outstr[maxlen-1]='\0'; - } else { - memcpy(outstr, url, translen); - outstr[translen]='\0'; - } - } - - return(NNTI_OK); -} - -NNTI_result_t nnti_url_get_address(const char *url, char *outstr, const int maxlen) -{ - int addrlen=0; - char *sep=NULL; - char *address=NULL; - - sep=strstr(url, "://"); - if (sep == NULL) { - /* invalid URL */ - return(NNTI_EINVAL); - } else { - address=sep+3; - sep=strchr(address, '/'); - if (sep == NULL) { - /* no trailing slash */ - addrlen=strlen(address); - } else { - addrlen=sep-address; - } - } - if (addrlen == 0) { - /* invalid URL */ - return(NNTI_EINVAL); - } - - if (addrlen >= maxlen) { - /* */ - memcpy(outstr, address, maxlen-1); - outstr[maxlen-1]='\0'; - } else { - memcpy(outstr, address, addrlen); - outstr[addrlen]='\0'; - } - - return(NNTI_OK); -} - -NNTI_result_t nnti_url_get_memdesc(const char *url, char *outstr, const int maxlen) -{ - int mdlen=0; - char *sep=NULL; - char *memdesc=NULL; - - sep=strstr(url, "://"); - if (sep == NULL) { - /* invalid URL */ - return(NNTI_EINVAL); - } else { - char *address=sep+3; - sep=strchr(address, '/'); - if (sep == NULL) { - /* no trailing slash, so no memory descriptor */ - outstr[0]='\0'; - return(NNTI_OK); - } else { - memdesc=sep+1; - mdlen=strlen(memdesc); - } - } - if (mdlen == 0) { - /* nothing after trailing slash, so no memory descriptor */ - outstr[0]='\0'; - return(NNTI_OK); - } - - if (mdlen >= maxlen) { - /* */ - memcpy(outstr, memdesc, maxlen-1); - outstr[maxlen-1]='\0'; - } else { - memcpy(outstr, memdesc, mdlen); - outstr[mdlen]='\0'; - } - - return(NNTI_OK); -} - -NNTI_result_t nnti_url_get_params(const char *url, char *outstr, const int maxlen) -{ - int plen=0; - char *sep=NULL; - char *params=NULL; - - sep=strstr(url, "://"); - if (sep == NULL) { - /* invalid URL */ - return(NNTI_EINVAL); - } else { - char *address=sep+3; - sep=strchr(address, '?'); - if (sep == NULL) { - /* no question mark, so no parameters */ - outstr[0]='\0'; - return(NNTI_OK); - } else { - params=sep+1; - plen=strlen(params); - } - } - if (plen == 0) { - /* nothing after question mark, so no params */ - outstr[0]='\0'; - return(NNTI_OK); - } - - if (plen >= maxlen) { - /* */ - memcpy(outstr, params, maxlen-1); - outstr[maxlen-1]='\0'; - } else { - memcpy(outstr, params, plen); - outstr[plen]='\0'; - } - - return(NNTI_OK); -} - -int nnti_sleep(const uint64_t msec) -{ - int rc=0; - struct timespec ts, rmtp; - - ts.tv_sec=0; - if (msec < 1000) { - ts.tv_nsec=msec*1000*1000; /* 1sec == 1ns*1000*1000*1000; */ - } else { - uint64_t sec=msec/1000; - ts.tv_sec=sec; - ts.tv_nsec=(msec-(sec*1000))*1000*1000; - } - - rc=nanosleep(&ts, &rmtp); - if (rc!=0) log_error(nnti_debug_level, "nanosleep failed: %s\n", strerror(errno)); - - return(rc); -} diff --git a/packages/trios/libraries/nessie/nnti/nnti_utils.h b/packages/trios/libraries/nessie/nnti/nnti_utils.h deleted file mode 100644 index cef1455fe1be..000000000000 --- a/packages/trios/libraries/nessie/nnti/nnti_utils.h +++ /dev/null @@ -1,71 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * nnti_utils.h - * - * Created on: Feb 16, 2011 - * Author: thkorde - */ - -#ifndef NNTI_UTILS_H_ -#define NNTI_UTILS_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -NNTI_result_t nnti_url_get_transport(const char *url, char *outstr, const int maxlen); -NNTI_result_t nnti_url_get_address(const char *url, char *outstr, const int maxlen); -NNTI_result_t nnti_url_get_memdesc(const char *url, char *outstr, const int maxlen); -NNTI_result_t nnti_url_get_params(const char *url, char *outstr, const int maxlen); - - -int nnti_sleep(const uint64_t msec); - -#ifdef __cplusplus -} -#endif - -#endif /* NNTI_UTILS_H_ */ diff --git a/packages/trios/libraries/nessie/nssi/CMakeLists.txt b/packages/trios/libraries/nessie/nssi/CMakeLists.txt deleted file mode 100644 index dde98b0bf609..000000000000 --- a/packages/trios/libraries/nessie/nssi/CMakeLists.txt +++ /dev/null @@ -1,125 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -SET(NSSI_HEADERS "") -SET(NSSI_NONINSTALLHEADERS "") -SET(NSSI_SOURCES "") - - -############# COMMON ########################## - -TRIBITS_SUBPACKAGE(nssi) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -############# NSSI ########################## - -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/Trios_nssi_types_xdr.x) -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/nssi_service_args.x) - -APPEND_SET(NSSI_NOINSTHEADERS - ${CMAKE_CURRENT_BINARY_DIR}/nssi_service_args.h - nssi_opcodes.h - nssi_trace.h -) - -APPEND_SET(NSSI_HEADERS - Trios_nssi_rpc.h - Trios_nssi_types.h - Trios_nssi_debug.h - Trios_nssi_fprint_types.h - Trios_nssi_xdr.h - Trios_nssi_TypeManip.h - ${CMAKE_CURRENT_BINARY_DIR}/Trios_nssi_types_xdr.h -) - -APPEND_SET(NSSI_SOURCES - ${CMAKE_CURRENT_BINARY_DIR}/Trios_nssi_types_xdr.c - ${CMAKE_CURRENT_BINARY_DIR}/nssi_service_args.c - fprint_types.cpp - nssi_common.cpp - nssi_debug.c - nssi_xdr.cpp -) - -############# CLIENT ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../client) - -APPEND_SET(NSSI_HEADERS - ../client/Trios_nssi_client.h - ../client/Trios_nssi_request.h -) - -APPEND_SET(NSSI_SOURCES - ../client/nssi_client.cpp -) - -############# SERVER ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../server) - -APPEND_SET(NSSI_HEADERS - ../server/Trios_nssi_server.h -) - -APPEND_SET(NSSI_SOURCES - ../server/nssi_server.cpp -) - -############# libtrios_nssi.a ########################## - -TRIBITS_ADD_LIBRARY( - trios_nssi - HEADERS ${NSSI_HEADERS} - NOINSTALLHEADERS ${NSSI_NOINSTALLHEADERS} - SOURCES ${NSSI_SOURCES} - DEPLIBS ${DEPLIBS} -) - - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_TypeManip.h b/packages/trios/libraries/nessie/nssi/Trios_nssi_TypeManip.h deleted file mode 100644 index ea1265dae035..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_TypeManip.h +++ /dev/null @@ -1,296 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -//////////////////////////////////////////////////////////////////////////////// -// The Loki Library -// Copyright (c) 2001 by Andrei Alexandrescu -// This code accompanies the book: -// Alexandrescu, Andrei. "Modern C++ Design: Generic Programming and Design -// Patterns Applied". Copyright (c) 2001. Addison-Wesley. -// Permission to use, copy, modify, distribute and sell this software for any -// purpose is hereby granted without fee, provided that the above copyright -// notice appear in all copies and that both that copyright notice and this -// permission notice appear in supporting documentation. -// The author or Addison-Welsey Longman make no representations about the -// suitability of this software for any purpose. It is provided "as is" -// without express or implied warranty. -//////////////////////////////////////////////////////////////////////////////// - -// Last update: November 22, 2001 - -#ifndef _TRIOS_NSSI_TYPEMANIP_H_ -#define _TRIOS_NSSI_TYPEMANIP_H_ - -namespace Nessie -{ -//////////////////////////////////////////////////////////////////////////////// -// class template Int2Type -// Converts each integral constant into a unique type -// Invocation: Int2Type where v is a compile-time constant integral -// Defines 'value', an enum that evaluates to v -//////////////////////////////////////////////////////////////////////////////// - -template -struct Int2Type -{ - enum { value = v }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// class template Type2Type -// Converts each type into a unique, insipid type -// Invocation Type2Type where T is a type -// Defines the type OriginalType which maps back to T -//////////////////////////////////////////////////////////////////////////////// - -template -struct Type2Type -{ - typedef T OriginalType; -}; - -//////////////////////////////////////////////////////////////////////////////// -// class template Select -// Selects one of two types based upon a boolean constant -// Invocation: Select::Result -// where: -// flag is a compile-time boolean constant -// T and U are types -// Result evaluates to T if flag is true, and to U otherwise. -//////////////////////////////////////////////////////////////////////////////// - -template -struct Select -{ - typedef T Result; -}; -template -struct Select -{ - typedef U Result; -}; - -//////////////////////////////////////////////////////////////////////////////// -// class template IsSameType -// Return true iff two given types are the same -// Invocation: SameType::value -// where: -// T and U are types -// Result evaluates to true iff U == T (types equal) -//////////////////////////////////////////////////////////////////////////////// - -template -struct IsSameType -{ - enum { value = false }; -}; - -template -struct IsSameType -{ - enum { value = true }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// Helper types Small and Big - guarantee that sizeof(Small) < sizeof(Big) -//////////////////////////////////////////////////////////////////////////////// - -namespace Private -{ -template -struct ConversionHelper -{ - typedef char Small; - struct Big { char dummy[2]; }; - static Big Test(...); - static Small Test(U); - static T MakeT(); -}; -} - -//////////////////////////////////////////////////////////////////////////////// -// class template Conversion -// Figures out the conversion relationships between two types -// Invocations (T and U are types): -// a) Conversion::exists -// returns (at compile time) true if there is an implicit conversion from T -// to U (example: Derived to Base) -// b) Conversion::exists2Way -// returns (at compile time) true if there are both conversions from T -// to U and from U to T (example: int to char and back) -// c) Conversion::sameType -// returns (at compile time) true if T and U represent the same type -// -// Caveat: might not work if T and U are in a private inheritance hierarchy. -//////////////////////////////////////////////////////////////////////////////// - -template -struct Conversion -{ - typedef Private::ConversionHelper H; -#ifndef __MWERKS__ - enum { exists = sizeof(typename H::Small) == sizeof((H::Test(H::MakeT()))) }; -#else - enum { exists = false }; -#endif - enum { exists2Way = exists && Conversion::exists }; - enum { sameType = false }; -}; - -template -struct Conversion -{ - enum { exists = 1, exists2Way = 1, sameType = 1 }; -}; - -template -struct Conversion -{ - enum { exists = 0, exists2Way = 0, sameType = 0 }; -}; - -template -struct Conversion -{ - enum { exists = 0, exists2Way = 0, sameType = 0 }; -}; - -template <> -struct Conversion -{ - public: - enum { exists = 1, exists2Way = 1, sameType = 1 }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// class template SuperSubclass -// Invocation: SuperSubclass::value where B and D are types. -// Returns true if B is a public base of D, or if B and D are aliases of the -// same type. -// -// Caveat: might not work if T and U are in a private inheritance hierarchy. -//////////////////////////////////////////////////////////////////////////////// - -template -struct SuperSubclass -{ - enum { value = (::Nessie::Conversion::exists && - !::Nessie::Conversion::sameType) }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// class template SuperSubclassStrict -// Invocation: SuperSubclassStrict::value where B and D are types. -// Returns true if B is a public base of D. -// -// Caveat: might not work if T and U are in a private inheritance hierarchy. -//////////////////////////////////////////////////////////////////////////////// - -template -struct SuperSubclassStrict -{ - enum { value = (::Nessie::Conversion::exists && - !::Nessie::Conversion::sameType && - !::Nessie::Conversion::sameType) }; -}; - -//////////////////////////////////////////////////////////////////////////////// -// class template SizeOf -//////////////////////////////////////////////////////////////////////////////// -template -class SizeOf -{ - public: - static char check(Type2Type); - static char check(Type2Type); - static char check(Type2Type); - static char check(Type2Type); - static T check(...); - - public: - enum {value = sizeof(check(Type2Type()))}; -}; - -} // namespace Nessie - -//////////////////////////////////////////////////////////////////////////////// -// macro SUPERSUBCLASS -// Invocation: SUPERSUBCLASS(B, D) where B and D are types. -// Returns true if B is a public base of D, or if B and D are aliases of the -// same type. -// -// Caveat: might not work if T and U are in a private inheritance hierarchy. -// Deprecated: Use SuperSubclass class template instead. -//////////////////////////////////////////////////////////////////////////////// - -#define SUPERSUBCLASS(T, U) \ - ::Nessie::SuperSubclass::value - -//////////////////////////////////////////////////////////////////////////////// -// macro SUPERSUBCLASS_STRICT -// Invocation: SUPERSUBCLASS(B, D) where B and D are types. -// Returns true if B is a public base of D. -// -// Caveat: might not work if T and U are in a private inheritance hierarchy. -// Deprecated: Use SuperSubclassStrict class template instead. -//////////////////////////////////////////////////////////////////////////////// - -#define SUPERSUBCLASS_STRICT(T, U) \ - ::Nessie::SuperSubclassStrict::value - -//////////////////////////////////////////////////////////////////////////////// -// Change log: -// June 20, 2001: ported by Nick Thurn to gcc 2.95.3. Kudos, Nick!!! -// November 22, 2001: minor change to support porting to boost -// November 22, 2001: fixed bug in Conversion -// (credit due to Brad Town) -// November 23, 2001: (well it's 12:01 am) fixed bug in SUPERSUBCLASS - added -// the volatile qualifier to be 100% politically correct -// September 16, 2002: Changed "const volatile" to "const volatile *", to enable -// conversion to succeed. Done earlier by MKH. -// Added SuperSubclass and SuperSubclassStrict templates. The corresponding -// macros are deprecated. -// Added extra parenthesis in sizeof in Conversion, to disambiguate function -// call from function declaration. T.S. -//////////////////////////////////////////////////////////////////////////////// - -#endif // _TRIOS_NSSI_TYPEMANIP_H_ diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_debug.h b/packages/trios/libraries/nessie/nssi/Trios_nssi_debug.h deleted file mode 100644 index 058c3c019370..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_debug.h +++ /dev/null @@ -1,68 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file rpc_debug.h - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - * - */ - -#ifndef _TRIOS_NSSI_DEBUG_H_ -#define _TRIOS_NSSI_DEBUG_H_ - -#include "Trios_logger.h" - -#ifdef __cplusplus -extern "C" { -#endif - - extern log_level rpc_debug_level; - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_fprint_types.h b/packages/trios/libraries/nessie/nssi/Trios_nssi_fprint_types.h deleted file mode 100644 index 95673c0743a7..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_fprint_types.h +++ /dev/null @@ -1,172 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file Trios_nssi_fprint_types.h - * - * @brief Pretty print the different NSSI types. - * - */ - -#ifndef _TRIOS_NSSI_FPRINT_TYPES_H_ -#define _TRIOS_NSSI_FPRINT_TYPES_H_ - -#include -#include "Trios_nssi_types.h" - - -#if defined(__STDC__) || defined(__cplusplus) - -#ifdef __cplusplus -extern "C" { -#endif - - extern const char* nssi_err_str(const int rc); - - /** - * @brief Print out a return code. - * - * @param fp @input_type The output file. - * @param name @input_type The name of the variable. - * @param prefix @input_type Text that precedes the variable. - * @param rc @input_type The return code. - */ - extern void fprint_nssi_return_code( - FILE *fp, - const char *name, - const char *prefix, - const int rc); - - - /** - * @brief Output the contents of a request header. - * - * @param fp @input_type File pointer (where to send output) - * @param name @input_type The name of the variable. - * @param prefix @input_type Text to put on every line before the output. - * @param hdr @input_type The request header. - */ - extern void fprint_nssi_request_header( - FILE *fp, - const char *name, - const char *prefix, - const nssi_request_header *hdr); - - /** - * @brief Output the contents of a result header. - * - * @param fp @input_type File pointer (where to send output) - * @param name @input_type The name of the variable. - * @param prefix @input_type Text to put on every line before the output. - * @param hdr @input_type The result header. - */ - extern void fprint_nssi_result_header( - FILE *fp, - const char *name, - const char *prefix, - const nssi_result_header *hdr); - - - extern void fprint_nssi_rpc_encode( - FILE *fp, - const char *name, - const char *prefix, - const nssi_rpc_encode *rpc_encode); - - extern void fprint_nssi_service( - FILE *fp, - const char *name, - const char *prefix, - const nssi_service *svc); - - extern void fprint_nssi_ssize( - FILE *fp, - const char *name, - const char *prefix, - const nssi_ssize *ssize); - -#ifdef __cplusplus -} -#endif - - /* this is only for C++ code */ -#ifdef __cplusplus -#include - - void fprint_nssi_return_code( - std::ostream &out, - const char *name, - const char *prefix, - const int rc); - extern void fprint_nssi_request_header( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_request_header *hdr); - extern void fprint_nssi_result_header( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_result_header *hdr); - extern void fprint_nssi_rpc_encode( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_rpc_encode *rpc_encode); - extern void fprint_nssi_service( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_service *svc); - extern void fprint_nssi_ssize( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_ssize *ssize); -#endif - -#else /* K&R C */ - - -#endif /* K&R C */ - -#endif diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_rpc.h b/packages/trios/libraries/nessie/nssi/Trios_nssi_rpc.h deleted file mode 100644 index aa71e4c55d83..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_rpc.h +++ /dev/null @@ -1,147 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file rpc_common.h - * - * @brief Prototypes for the RPC methods used by both clients and servers. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * @author Rolf Riesen (rolf\@cs.sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - */ - -#ifndef _TRIOS_NSSI_RPC_H_ -#define _TRIOS_NSSI_RPC_H_ - -#include "Trios_config.h" -#include "Trios_nssi_types.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(HAVE_TRIOS_INFINIBAND) -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_IB -#elif defined(HAVE_TRIOS_GEMINI) -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_GEMINI -#elif defined(HAVE_TRIOS_BGPDCMF) -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_BGPDCMF -#elif defined(HAVE_TRIOS_BGQPAMI) -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_BGQPAMI -#elif defined(HAVE_TRIOS_MPI) -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_MPI -#elif defined(HAVE_TRIOS_PORTALS) || defined(HAVE_TRIOS_CRAYPORTALS) -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_PTL -#else -#define NSSI_DEFAULT_TRANSPORT NSSI_RPC_LOCAL -#endif - -#define NSSI_DEFAULT_ENCODE NSSI_RPC_XDR - -#if defined(__STDC__) || defined(__cplusplus) - - extern void *memdup(void *src, int size); - - /** - * @brief Initialize the RPC mechanism. - * - * The \ref nssi_rpc_init method - * initializes the rpc library. - * - * @param rpc_transport @input_type Identifies the transport mechanism - * to use for communication. - * @param rpc_encode @input_type Identifies the mechanism used to - * encode the rpc control messages. - * - * @return \ref NSSI_OK Indicates sucess. - * @return \ref NSSI_ERR_RPC Indicates failure in the NSSI RPC library. - */ - extern int nssi_rpc_init( - const nssi_rpc_transport rpc_transport, - const nssi_rpc_encode rpc_encode, - const char *url); - - /** - * @brief Finalize the RPC mechanism. - * - * The \b nssi_rpc_fini method performs any cleanup - * required by the underlying communication protocol - * before exiting the client application. - * - * @return \ref NSSI_OK Indicates sucess. - * @return \ref NSSI_ERR_RPC Indicates failure in the NSSI RPC library. - */ - extern int nssi_rpc_fini(const nssi_rpc_transport rpc_transport); - - /** - * @brief Get the process ID of this process. - * - * @ingroup rpc_funcs - * - * The \ref nssi_get_my_pid method gets the - * \ref nssi_remote_pid "process id" of the calling process. - * - * @param id @output_type If successful, points to the \ref nssi_remote_pid "process ID" - * of the calling process. Undefined otherwise. - * - * @return \ref NSSI_OK Indicates sucess. - * @return \ref NSSI_ERR_RPC Indicates failure in the NSSI RPC library. - * - */ - extern int nssi_get_url( - const nssi_rpc_transport rpc_transport, - char *url, - uint32_t maxlen); - -#else /* K&R C */ -#endif - - - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_types.h b/packages/trios/libraries/nessie/nssi/Trios_nssi_types.h deleted file mode 100644 index 3fa9994e2e42..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_types.h +++ /dev/null @@ -1,128 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file Trios_nssi_types.h - * - * @brief Type definitions and method prototypes for - * the LWFS. - * - * This file includes the necessary data structures - * required by an application that uses the LWFS. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1064 $ - * $Date: 2007-01-19 10:05:13 -0700 (Fri, 19 Jan 2007) $ - * - */ - -#ifndef _TRIOS_NSSI_TYPES_H_ -#define _TRIOS_NSSI_TYPES_H_ - -#include "Trios_config.h" - -#include - -/* If sockaddr_in is not defined, create some bogus - * definition. (We don't use it anyway, but sys/socket.h - * references it. This works around a Cray bug. - */ -#ifndef HAVE_TRIOS_STRUCT_SOCKADDR_IN -#ifndef HAVE_STRUCT_SOCKADDR_IN -#define HAVE_STRUCT_SOCKADDR_IN -struct sockaddr_in { - int a; -}; -#endif -#endif - - - -typedef struct { - uint8_t put_data_in_request; - - uint8_t use_buffer_queue; - - uint32_t buffer_queue_initial_size; - uint32_t buffer_queue_max_size; - uint8_t buffer_queue_create_if_empty; - - uint32_t rdma_buffer_queue_initial_size; - uint32_t rdma_buffer_queue_max_size; - uint8_t rdma_buffer_queue_create_if_empty; - uint32_t rdma_buffer_queue_buffer_size; -} nssi_config_t; - - - -/* Treat the rpc-generated includes as system includes to avoid warnings */ -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define UUID_TO_UINT64(u) \ - ((uint64_t)( \ - (*((uint64_t *)&(u[0]))) \ - ^(*((uint64_t *)&(u[8]))) )) - -#define UUID_TO_UINT32(u) \ - ((uint32_t)(\ - (*((uint32_t *)&(u[0]))) \ - ^(*((uint32_t *)&(u[4]))) \ - ^(*((uint32_t *)&(u[8]))) \ - ^(*((uint32_t *)&(u[12]))) )) - - -#if defined(__STDC__) || defined(__cplusplus) - -#else /* K&R C */ - -#endif /* K&R C */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_types_xdr.x b/packages/trios/libraries/nessie/nssi/Trios_nssi_types_xdr.x deleted file mode 100644 index 309ea00bc527..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_types_xdr.x +++ /dev/null @@ -1,343 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file nssi_types_xdr.x - * - * @brief XDR definitions for types used in the NSSI RPC interface. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * - */ - -#ifdef RPC_HDR -%#include "Trios_xdr.h" -%#include "Trios_nnti_xdr.h" -#endif - -#ifdef RPC_XDR -%#include "Trios_xdr.h" -%#include "Trios_nnti_xdr.h" -#endif - -/** @addtogroup base_types - * @{ - */ - -/** - * - * @brief NSSI Return Codes. - * - * Each NSSI function returns an integer value that corresponds - * to an error, a warning, or successfull completion. These values are analogous - * to the values from the UNIX include file ``error.h''. - */ -enum nssi_return_codes { - - /** @brief The function completed successfully. */ - NSSI_OK = 0, - - /** @brief Unspecified I/O error. */ - NSSI_EIO = 1001, - - /** @brief The size of the message is larger than the supported maximum size. */ - NSSI_EMSGSIZE, - - /** @brief The operation or process has been canceled. */ - NSSI_ECANCELED, - - /** @brief An operation timed out. */ - NSSI_ETIMEDOUT, - - /** @brief The value of the variable or parameter is invalid. */ - NSSI_EINVAL, - - /** @brief No memory is available. */ - NSSI_ENOMEM, - - /** @brief No such entry. */ - NSSI_ENOENT, - - /** @brief Unsupported operation. */ - NSSI_ENOTSUP, - - /** @brief The item already exists. */ - NSSI_EEXIST, - - /** @brief Unsuccessful RPC operation. */ - NSSI_EBADRPC, - - /** @brief Not initialized. */ - NSSI_ENOTINIT, - - /** @brief Insufficient priveleges to perform operation. */ - NSSI_EPERM, - - /** @brief Error decoding an RPC request. */ - NSSI_EDECODE, - - /** @brief Error encoding an RPC request. */ - NSSI_EENCODE, - - /** @brief An operation would have blocked. */ - NSSI_EWOULDBLOCK, - - /** @brief Operation was interupted, but possibly recoverable. */ - NSSI_EAGAIN - -}; - - -/** - * @brief The \ref nssi_size type is used to for ``size'' variables. - */ -typedef uint64_t nssi_size; -typedef uint64_t nssi_ssize; - - - -/** - * @brief Length of an nssi_name. - */ -const NSSI_HOSTNAME_LEN = NNTI_HOSTNAME_LEN; -const NSSI_URL_LEN = NNTI_URL_LEN; - - - - -/** - * @brief Enumerator for the different type of transport mechanisms. - * - * The \ref nssi_rpc_transport enumerator provides integer values - * to represent the different types of supported transport mechanisms. - * Initially, the only supported transport mechanism is Portals. - */ -enum nssi_rpc_transport { - /** @brief No operations permitted. */ - NSSI_RPC_NULL, - - /** @brief Use Portals to transfer rpc requests. */ - NSSI_RPC_PTL, - - /** @brief Use Infiniband to transfer rpc requests. */ - NSSI_RPC_IB, - - /** @brief Use Cray Gemini to transfer rpc requests. */ - NSSI_RPC_GEMINI, - - /** @brief Use Blue Gene/P DCMF Lib to transfer rpc requests. */ - NSSI_RPC_BGPDCMF, - - /** @brief Use Blue Gene/P PAMI Lib to transfer rpc requests. */ - NSSI_RPC_BGQPAMI, - - /** @brief Use MPI to transfer rpc requests. */ - NSSI_RPC_MPI, - - /** @brief Use a local buffer (not a remote operation). */ - NSSI_RPC_LOCAL -}; - -/** - * @brief The number of RPC mechanisms supported by NSSI. - */ -const NSSI_RPC_COUNT = 8; - - -/** - * @brief Enumerator for the different type of encoding mechanisms. - * - * The \ref nssi_rpc_encode enumerator provides integer values - * to represent the different types of supported mechanisms for encoding - * control messages transferred to/from NSSI servers. - * Initially, the only supported transport mechanism is XDR. - */ -enum nssi_rpc_encode { - /** @brief Use XDR to encode/decode rpc requests and results. */ - NSSI_RPC_XDR -}; - - -/** - * @brief Default timeout for rpc calls (in ms). - */ -const DEFAULT_RPC_TIMEOUT = 10000000; - - -/** - * @brief A descriptor for remote services. - * - * The \ref nssi_service data - * structure contains information needed by the client - * to send operation requests to a remote service, including the process ID - * of the remote host, the encoding mechanism to use for control messages, - * and the memory location reserved for incoming requests. - * - * A client obtains the \ref nssi_service structure by calling the - * \ref nssi_get_service "nssi_get_service()" function. - */ -struct nssi_service { - - /** @brief Identifies the RPC mechanism to use for transfering messages. */ - NNTI_transport_id_t transport_id; - - /** @brief Identifies the mechanism to use for encoding messages. */ - nssi_rpc_encode rpc_encode; - - /** @brief The address of the service. */ - NNTI_peer_t svc_host; - - /** @brief The remote memory address reserved for incoming requests. */ - NNTI_buffer_t req_addr; - - /** @brief This service accepts requests up to 'req_size' is length. */ - int32_t req_size; - - /** @brief This service sends results up to 'res_size' is length. */ - int32_t res_size; - - /** @brief The maximum number of requests to process. */ - int32_t max_reqs; - - /** @brief A callback to invoke at intervals defined by progess_callback_timeout. */ - uint64_t progress_callback; - - /** @brief The interval at which progess_callback should be invoked. Note: This is not a real-time environment, so there is no guarentee of the interval. */ - uint64_t progress_callback_timeout; -}; - -/** - * @brief The size of an encoded nssi_request_header buffer. - */ -const NSSI_SHORT_REQUEST_SIZE = NNTI_REQUEST_BUFFER_SIZE; -/** - * @brief The size of an encoded nssi_result_header buffer. - */ -const NSSI_SHORT_RESULT_SIZE = NNTI_RESULT_BUFFER_SIZE; - - -/** - * @brief The request header. - * - * The nssi_request_header structure contains details needed by an - * NSSI server to perform a remote operation. - * - * The operation arguments may or may not be sent to the server - * in the original request. If the field \em fetch_args is - * \em true, the server will fetch arguments from \em args_addr - * in a separate operation. Otherwise, the client sends the - * arguments to the server with the request. - */ -struct nssi_request_header { - NNTI_transport_header_t transport_header; - - /** @brief ID of the request */ - uint32_t id; - - /** @brief ID of the operation to perform. */ - uint32_t opcode; - - /** @brief A flag that tells the server to fetch args from - * \ref args_addr. */ - bool fetch_args; - - /** @brief A flag that tells the server to fetch bulk data from - * \ref data_addr. */ - bool fetch_data; - - /** @brief A flag that tells the server if the client expects - * a response. */ - bool is_responseless; - - /** @brief The remote memory address reserved for - * long arguments. */ - NNTI_buffer_t args_addr; - - /** @brief The remote memory address reserved for bulk - * data transfers. */ - NNTI_buffer_t data_addr; - - /** @brief The remote memory address reserved for the short result. */ - NNTI_buffer_t res_addr; -}; - - -/** - * @brief The result header. - * - * The mds_request_header structure contains the details needed by the - * MDS server to perform a remote operation. - * - * The arguments may or may not be sent to the server in the original request. - * If the field \em fetchargs is \em true, the arguments will be fetched - * from \em args_portal in a separate operation by the server. Otherwise, - * the arguments are sent directly to the server. - * - * NO POINTERS ALLOWED IN THIS STRUCTURE! - * - */ -struct nssi_result_header { - NNTI_transport_header_t transport_header; - - /** @brief ID of the result (should be same as request) */ - uint32_t id; - - /** @brief ID of the operation to perform. */ - uint32_t opcode; - - /** @brief Size of the result. */ - uint32_t result_size; - - /** @brief A flag that tells the client to "get" result from the client. */ - bool fetch_result; - - /** @brief The remote memory address reserved for long results. */ - NNTI_buffer_t result_addr; - - /** @brief The remote memory address reserved for long results ACK. */ - NNTI_buffer_t result_ack_addr; - - /** @brief The return code of the function. */ - uint32_t rc; -}; - -/** @} */ diff --git a/packages/trios/libraries/nessie/nssi/Trios_nssi_xdr.h b/packages/trios/libraries/nessie/nssi/Trios_nssi_xdr.h deleted file mode 100644 index 2bb904bd88be..000000000000 --- a/packages/trios/libraries/nessie/nssi/Trios_nssi_xdr.h +++ /dev/null @@ -1,150 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file Trios_nssi_xdr.h - * - * @brief Prototypes for the RPC methods used by both clients and servers. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * @author Rolf Riesen (rolf\@cs.sandia.gov) - * $Revision: 301 $ - * $Date: 2005-03-16 00:21:00 -0700 (Wed, 16 Mar 2005) $ - */ - -#ifndef _TRIOS_NSSI_XDR_H_ -#define _TRIOS_NSSI_XDR_H_ - -#include "Trios_config.h" -#include "Trios_xdr.h" - - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - -/** - * @brief Initialize the XDR encoding mechanism. - */ -extern int nssi_xdr_init(); - -/** - * @brief Finalize the XDR encoding mechanism. - */ -extern int nssi_xdr_fini(); - -/** - * @brief Register encodings for standard service ops. - */ -extern int register_service_encodings(void); - -/** - * @brief Register xdr encodings for an opcode. - * - * The \ref nssi_register_xdr_encodings function registers - * the xdr encodings functions associated with a particular remote - * NSSI operation. - * - * @param opcode @input_type The opcode to lookup. - * @param encode_args @input_type The xdr function used to - * encode/decode the arguments. - * @param encode_data @input_type The xdr function used to - * encode/decode the data. - * @param encode_result @input_type The xdr function used to - * encode/decode the result. - */ -extern int nssi_register_xdr_encoding(int opcode, - xdrproc_t encode_args, - xdrproc_t encode_data, - xdrproc_t encode_result); - - -/** - * @brief Lookup the encodings for an opcode. - * - * The \ref nssi_lookup_xdr_encodings function looks up - * the registered xdr encodings for an opcode. - * - * @param opcode @input_type The opcode to lookup. - * @param encode_args @output_type Points to the xdr function used to - * encode/decode the arguments. - * @param encode_data @output_type Points to the xdr function used to - * encode/decode the data. - * @param encode_result @output_type Points to the xdr function used to - * encode/decode the result. - * - * @return \ref NSSI_OK If successful. - * @return \ref NSSI_ERR_NOENT If the encodings for the specified - * opcode do not exist. - */ -int nssi_lookup_xdr_encoding(int opcode, - xdrproc_t *encode_args, - xdrproc_t *encode_data, - xdrproc_t *encode_result); - - -/** - * @brief Remove all encodings. - * - * The \ref nssi_clear_xdr_encodings function removes - * the xdr encodings functions associated with all remote - * NSSI operations. - * - */ -int nssi_clear_xdr_encodings(); - -#else /* K&R C */ -#endif - - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/packages/trios/libraries/nessie/nssi/cmake/Dependencies.cmake b/packages/trios/libraries/nessie/nssi/cmake/Dependencies.cmake deleted file mode 100644 index 2f3c3c01ac05..000000000000 --- a/packages/trios/libraries/nessie/nssi/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnnti) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/libraries/nessie/nssi/fprint_types.cpp b/packages/trios/libraries/nessie/nssi/fprint_types.cpp deleted file mode 100644 index 5354205311c4..000000000000 --- a/packages/trios/libraries/nessie/nssi/fprint_types.cpp +++ /dev/null @@ -1,466 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file fprint_types.c - * - * @brief Pretty print the different NSSI types. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 1640 $. - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $. - * - */ - -#include -#include /* find memcpy() */ -#include -#include - -#include "Trios_logger.h" -#include "Trios_nssi_types.h" -#include "Trios_nnti_fprint_types.h" -#include "Trios_nssi_fprint_types.h" - -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE -#endif -#include - - - -/* ----------- Utility functions to output types ---- */ - - - -static const char *myitoa(int val) { - static char buf[32]; - - snprintf(buf, sizeof(buf), "%d", val); - return buf; -} - -/** - * @brief Output a string associated with a return code. - * - * @param rc @input the return code. - * - * @returns A string associated with the return code. - */ -const char *nssi_err_str(int rc) -{ - switch (rc) { - - case NSSI_OK: - return "NSSI_OK"; - - /** @brief Unspecified I/O error. */ - case NSSI_EIO: - return "NSSI_EIO"; - - /** @brief The size of the message is larger than the supported maximum size. */ - case NSSI_EMSGSIZE: - return "NSSI_EMSGSIZE"; - - /** @brief The operation or process has been canceled. */ - case NSSI_ECANCELED: - return "NSSI_ECANCELED"; - - /** @brief An operation timed out. */ - case NSSI_ETIMEDOUT: - return "NSSI_ETIMEDOUT"; - - /** @brief The value of the variable or parameter is invalid. */ - case NSSI_EINVAL: - return "NSSI_EINVAL"; - - /** @brief No memory is available. */ - case NSSI_ENOMEM: - return "NSSI_ENOMEM"; - - /** @brief No such entry. */ - case NSSI_ENOENT: - return "NSSI_ENOENT"; - - /** @brief Unsupported operation. */ - case NSSI_ENOTSUP: - return "NSSI_ENOTSUP"; - - /** @brief The item already exists. */ - case NSSI_EEXIST: - return "NSSI_EEXIST"; - - /** @brief Unsuccessful RPC operation. */ - case NSSI_EBADRPC: - return "NSSI_EBADRPC"; - - /** @brief Not initialized. */ - case NSSI_ENOTINIT: - return "NSSI_ENOTINIT"; - - /** @brief Insufficient priveleges to perform operation. */ - case NSSI_EPERM: - return "NSSI_EPERM"; - - /** @brief Error decoding an RPC request. */ - case NSSI_EDECODE: - return "NSSI_EDECODE"; - - /** @brief Error encoding an RPC request. */ - case NSSI_EENCODE: - return "NSSI_EENCODE"; - - /** @brief An operation would have blocked. */ - case NSSI_EWOULDBLOCK: - return "NSSI_EWOULDBLOCK"; - - /** @brief Operation was interupted, but possibly recoverable. */ - case NSSI_EAGAIN: - return "NSSI_EAGAIN"; - - default: - return myitoa(rc); - } -} - - -/** - * @brief Print out a return code. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The return code. - */ -void fprint_nssi_return_code( - std::ostream &out, - const char *name, - const char *prefix, - const int rc) -{ - out << prefix << " " << name << " = " << nssi_err_str(rc) << std::endl; -} - -/** - * @brief Print out a return code. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The return code. - */ -void fprint_nssi_return_code( - FILE *fp, - const char *name, - const char *prefix, - const int rc) -{ - std::stringstream out(std::stringstream::out); - - fprint_nssi_return_code(out, name, prefix, rc); - - fprintf(fp, "%s", out.str().c_str()); -} - - -/** - * @brief Output the contents of a request header. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param hdr The request header. - */ -void fprint_nssi_request_header( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_request_header *hdr) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - if (hdr == NULL) { - out << prefix << " " << name << " = NULL " << std::endl; - return; - } - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - out << subprefix << " id = " << hdr->id << std::endl; - out << subprefix << " opcode = " << hdr->opcode << std::endl; - out << subprefix << " fetch_args = " << hdr->fetch_args << std::endl; - fprint_NNTI_buffer(out, "args_addr", subprefix.c_str(), &hdr->args_addr); - fprint_NNTI_buffer(out, "data_addr", subprefix.c_str(), &hdr->data_addr); - fprint_NNTI_buffer(out, "res_addr", subprefix.c_str(), &hdr->res_addr); - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a request header. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param hdr The request header. - */ -void fprint_nssi_request_header( - FILE *fp, - const char *name, - const char *prefix, - const nssi_request_header *hdr) -{ - std::stringstream out(std::stringstream::out); - - fprint_nssi_request_header(out, name, prefix, hdr); - - fprintf(fp, "%s", out.str().c_str()); -} - - -/** - * @brief Output the contents of a result header. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param hdr The result header. - */ -void fprint_nssi_result_header( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_result_header *hdr) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - if (hdr == NULL) { - out << prefix << " " << name << " = NULL " << std::endl; - return; - } - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - out << subprefix << " id = " << hdr->id << std::endl; - out << subprefix << " opcode = " << hdr->opcode << std::endl; - out << subprefix << " fetch_result = " << hdr->fetch_result << std::endl; - out << subprefix << " result_size = " << hdr->result_size << std::endl; - fprint_NNTI_buffer(out, "res_addr", subprefix.c_str(), &hdr->result_addr); - fprint_NNTI_buffer(out, "res_ack_addr", subprefix.c_str(), &hdr->result_ack_addr); - out << subprefix << " rc = " << hdr->rc << std::endl; - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Output the contents of a result header. - * - * @param fp File pointer (where to send output) - * @param prefix Text to put on every line before the output. - * @param hdr The result header. - */ -void fprint_nssi_result_header( - FILE *fp, - const char *name, - const char *prefix, - const nssi_result_header *hdr) -{ - std::stringstream out(std::stringstream::out); - - fprint_nssi_result_header(out, name, prefix, hdr); - - fprintf(fp, "%s", out.str().c_str()); -} - - -/** - * @brief Print out nssi_rpc_encode. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The encoding. - */ -void fprint_nssi_rpc_encode( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_rpc_encode *rpc_encode) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - /* contents */ - switch (*rpc_encode) { - case NSSI_RPC_XDR: - out << subprefix << " " << name << " = NSSI_RPC_XDR" << std::endl; - break; - - default: - out << subprefix << " " << name << " = UNDEFINED" << std::endl; - break; - } -} - -/** - * @brief Print out nssi_rpc_encode. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The encoding. - */ -void fprint_nssi_rpc_encode( - FILE *fp, - const char *name, - const char *prefix, - const nssi_rpc_encode *rpc_encode) -{ - std::stringstream out(std::stringstream::out); - - fprint_nssi_rpc_encode(out, name, prefix, rpc_encode); - - fprintf(fp, "%s", out.str().c_str()); -} - - -/** - * @brief Print out an nssi service descriptor. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The service. - */ -void fprint_nssi_service( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_service *svc) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - if (svc == NULL) { - out << prefix << " " << name << " = NULL " << std::endl; - return; - } - - /* header */ - out << prefix << " " << name << " = {" << std::endl; - - /* contents */ - fprint_nssi_rpc_encode(out, "rpc_encode", subprefix.c_str(), &(svc->rpc_encode)); - fprint_NNTI_peer(out, "svc_host", subprefix.c_str(), &(svc->svc_host)); - fprint_NNTI_buffer(out, "req_addr", subprefix.c_str(), &(svc->req_addr)); - out << subprefix << " max_reqs = " << svc->max_reqs << std::endl; - - /* footer */ - out << subprefix << " }" << std::endl; -} - -/** - * @brief Print out an nssi service descriptor. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The service. - */ -void fprint_nssi_service( - FILE *fp, - const char *name, - const char *prefix, - const nssi_service *svc) -{ - std::stringstream out(std::stringstream::out); - - fprint_nssi_service(out, name, prefix, svc); - - fprintf(fp, "%s", out.str().c_str()); -} - -/** - * @brief Print out an ssize. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The oid. - */ -void fprint_nssi_ssize( - std::ostream &out, - const char *name, - const char *prefix, - const nssi_ssize *ssize) -{ - std::string subprefix(prefix); - subprefix.append(" "); // add two spaces - - out << subprefix << " " << name << " = " << *ssize << std::endl; -} - -/** - * @brief Print out an ssize. - * - * @param fp The output file. - * @param name The name of the variable. - * @param prefix Text that precedes the variable. - * @param The oid. - */ -void fprint_nssi_ssize( - FILE *fp, - const char *name, - const char *prefix, - const nssi_ssize *ssize) -{ - std::stringstream out(std::stringstream::out); - - fprint_nssi_ssize(out, name, prefix, ssize); - - fprintf(fp, "%s", out.str().c_str()); -} diff --git a/packages/trios/libraries/nessie/nssi/nssi_common.cpp b/packages/trios/libraries/nessie/nssi/nssi_common.cpp deleted file mode 100644 index 4bd713e347f2..000000000000 --- a/packages/trios/libraries/nessie/nssi/nssi_common.cpp +++ /dev/null @@ -1,472 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file rpc_common.c - * - * @brief Implementation of the \ref rpc_client_api "RPC Client API". - * for the NSSI. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - * - */ - -#include "Trios_config.h" -#include "Trios_nssi_rpc.h" - -#include -#include -#include -#include - -#ifdef HAVE_TRIOS_MALLOC_H -#include -#endif - -/* string.h is required on Redsky to get the memcpy() declaration. */ -#include - -#include "Trios_nssi_types.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - -#include "Trios_threads.h" -#include "Trios_timer.h" -#include "Trios_logger.h" -#include "Trios_signal.h" -#include "buffer_queue.h" -#include "Trios_nssi_xdr.h" - -#include "Trios_nssi_debug.h" - -#include "Trios_nnti.h" - - -nssi_config_t nssi_config; -static void config_init(nssi_config_t *c); -static void config_get_from_env(nssi_config_t *c); - - -/* --------------------- Private methods ------------------- */ - -NNTI_transport_t transports[NSSI_RPC_COUNT]; - -static bool rpc_initialized = false; -static nssi_rpc_encode encoding = NSSI_DEFAULT_ENCODE; - -#define BQ_MIN 50 -#define BQ_MAX 1000 -trios_buffer_queue_t send_bq; -trios_buffer_queue_t recv_bq; -trios_buffer_queue_t rdma_target_bq; -trios_buffer_queue_t rdma_get_bq; -trios_buffer_queue_t rdma_put_bq; - - -void *memdup(void *src, int size) -{ - log_debug(rpc_debug_level, "enter"); - - void *new_buf = malloc(size*sizeof(char)); - memcpy(new_buf, src, size); - - return(new_buf); -} - -/** - * @brief Initialize the NSSI RPC mechanism. - * - * This implementation of \b nssi_rpc_init initializes the - * Portals library and interface so that we can use Portals - * as the underlying communication protocol. - * - * @param pid @input The ID to use for this process. - */ -int nssi_rpc_init( - const nssi_rpc_transport rpc_transport, - const nssi_rpc_encode rpc_encode, - const char *url) -{ - int rc; - - NNTI_transport_id_t transport_id; - - log_debug(rpc_debug_level, "************************************************************ entered"); - - if (rpc_initialized) return NSSI_OK; - - /* check to see if logging is enabled */ - if (logger_not_initialized()) { - logger_set_file(stderr); - } - - /* make sure the timer works properly */ -// rc = nssi_timer_test(); -// if (rc != NSSI_OK) { -// log_fatal(rpc_debug_level, "nssi_timer_test() failed"); -// return rc; -// } - - /* stash the transport and encoding types so we know how to cleanup on exit */ -// transport = rpc_transport; - encoding = rpc_encode; - - /* initialize the transport mechanism */ - switch (rpc_transport) { - case NSSI_RPC_PTL: - transport_id=NNTI_TRANSPORT_PORTALS; - break; - case NSSI_RPC_IB: - transport_id=NNTI_TRANSPORT_IB; - break; - case NSSI_RPC_GEMINI: - transport_id=NNTI_TRANSPORT_GEMINI; - break; - case NSSI_RPC_BGPDCMF: - transport_id=NNTI_TRANSPORT_DCMF; - break; - case NSSI_RPC_BGQPAMI: - transport_id=NNTI_TRANSPORT_PAMI; - break; - case NSSI_RPC_MPI: - transport_id=NNTI_TRANSPORT_MPI; - break; - default: - rc = NSSI_ENOENT; - log_error(rpc_debug_level, "the transport scheme %d does not exist", rpc_transport); - return rc; - } - - rc = NNTI_init( - transport_id, - url, - &transports[rpc_transport]); - if (rc != NNTI_OK) { - log_fatal(rpc_debug_level,"failed"); - return rc; - } - - /* initialize the xdr-encoding mechanism */ - switch (rpc_encode) { - case NSSI_RPC_XDR: - rc = nssi_xdr_init(); - if (rc != NSSI_OK) { - log_fatal(rpc_debug_level,"failed, %d", rc); - return rc; - } - break; - - default: - rc = NSSI_ENOENT; - log_error(rpc_debug_level, "the transport scheme " - "does not exist"); - return rc; - } - - config_init(&nssi_config); - config_get_from_env(&nssi_config); - if (nssi_config.use_buffer_queue) { - trios_buffer_queue_init( - &send_bq, - nssi_config.buffer_queue_initial_size, - nssi_config.buffer_queue_max_size, - nssi_config.buffer_queue_create_if_empty, - &transports[rpc_transport], - NNTI_SEND_SRC, - NSSI_SHORT_REQUEST_SIZE); - trios_buffer_queue_init( - &recv_bq, - nssi_config.buffer_queue_initial_size, - nssi_config.buffer_queue_max_size, - nssi_config.buffer_queue_create_if_empty, - &transports[rpc_transport], - NNTI_RECV_DST, - NSSI_SHORT_RESULT_SIZE); - trios_buffer_queue_init( - &rdma_target_bq, - nssi_config.rdma_buffer_queue_initial_size, - nssi_config.rdma_buffer_queue_max_size, - nssi_config.rdma_buffer_queue_create_if_empty, - &transports[rpc_transport], - (NNTI_buf_ops_t)(NNTI_GET_SRC|NNTI_PUT_DST), - nssi_config.rdma_buffer_queue_buffer_size); - trios_buffer_queue_init( - &rdma_get_bq, - nssi_config.rdma_buffer_queue_initial_size, - nssi_config.rdma_buffer_queue_max_size, - nssi_config.rdma_buffer_queue_create_if_empty, - &transports[rpc_transport], - NNTI_GET_DST, - nssi_config.rdma_buffer_queue_buffer_size); - trios_buffer_queue_init( - &rdma_put_bq, - nssi_config.rdma_buffer_queue_initial_size, - nssi_config.rdma_buffer_queue_max_size, - nssi_config.rdma_buffer_queue_create_if_empty, - &transports[rpc_transport], - NNTI_PUT_SRC, - nssi_config.rdma_buffer_queue_buffer_size); - } - - rpc_initialized = true; - - if (logging_debug(rpc_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "transports[rpc_transport].me", - "end of nssi_rpc_init", &transports[rpc_transport].me); - } - - log_debug(rpc_debug_level, "************************************************************ exit"); - - return NSSI_OK; -} - - -/** - * @brief Get the process ID of this process. - * - * @ingroup rpc_funcs - * - * The \ref nssi_get_my_pid method gets the - * \ref nssi_remote_pid "process id" of the calling process. - * - * @param id @output If successful, points to the \ref nssi_remote_pid "process ID" - * of the calling process. Undefined otherwise. - * - * @return \ref NSSI_OK Indicates sucess. - * @return \ref NSSI_ERR_RPC Indicates failure in the NSSI RPC library. - * - */ -int nssi_get_url( - const nssi_rpc_transport rpc_transport, - char *url, - uint32_t maxlen) -{ - NNTI_result_t rc=NNTI_OK; - - if (!rpc_initialized) { - log_error(rpc_debug_level, "RPC not initialized"); - return NSSI_ENOTINIT; - } - - rc=NNTI_get_url( - &transports[rpc_transport], - url, - maxlen); - - return(rc); -} - - -/** - * @brief Finalize the RPC mechanism. - * - * The \b nssi_rpc_finalize method performs any cleanup - * required by the underlying communication protocol - * before exiting the client application. - */ -int nssi_rpc_fini(const nssi_rpc_transport rpc_transport) -{ - int rc; - - if (nssi_config.use_buffer_queue) { - trios_buffer_queue_fini( - &send_bq); - trios_buffer_queue_fini( - &recv_bq); - } - - /* initialize the transport mechanism */ - rc = NNTI_fini(&transports[rpc_transport]); - if (rc != NNTI_OK) { - log_fatal(rpc_debug_level,"failed"); - return rc; - } - - /* initialize the xdr-encoding mechanism */ - switch (encoding) { - case NSSI_RPC_XDR: - rc = nssi_xdr_fini(); - if (rc != NSSI_OK) { - log_fatal(rpc_debug_level,"failed, %d", rc); - return rc; - } - break; - - default: - rc = NSSI_ENOENT; - log_error(rpc_debug_level, "the transport scheme " - "does not exist"); - return rc; - } - - rpc_initialized=false; - - return NSSI_OK; -} - -static void config_init(nssi_config_t *c) -{ - c->put_data_in_request=false; - c->use_buffer_queue =false; - c->buffer_queue_initial_size =50; - c->buffer_queue_max_size =1000; - c->buffer_queue_create_if_empty=true; - c->rdma_buffer_queue_initial_size =5; - c->rdma_buffer_queue_max_size =10; - c->rdma_buffer_queue_create_if_empty=true; - c->rdma_buffer_queue_buffer_size =0; /*1*1024*1024; /* 1 megabyte */ -} -static void config_get_from_env(nssi_config_t *c) -{ - char *env_str=NULL; - - if ((env_str=getenv("TRIOS_NSSI_PUT_DATA_IN_REQUEST")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(rpc_debug_level, "setting c->put_data_in_request to TRUE"); - c->put_data_in_request=true; - } else { - log_debug(rpc_debug_level, "setting c->put_data_in_request to FALSE"); - c->put_data_in_request=false; - } - } else { - log_debug(rpc_debug_level, "TRIOS_NNTI_PUT_DATA_IN_REQUEST is undefined. using c->put_data_in_request default"); - } - if ((env_str=getenv("TRIOS_NSSI_USE_BUFFER_QUEUE")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(rpc_debug_level, "setting c->use_buffer_queue to TRUE"); - c->use_buffer_queue=true; - } else { - log_debug(rpc_debug_level, "setting c->use_buffer_queue to FALSE"); - c->use_buffer_queue=false; - } - } else { - log_debug(rpc_debug_level, "TRIOS_NNTI_USE_BUFFER_QUEUE is undefined. using c->use_buffer_queue default"); - } - if ((env_str=getenv("TRIOS_NSSI_BUFFER_QUEUE_INITIAL_SIZE")) != NULL) { - errno=0; - uint32_t initial_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(rpc_debug_level, "setting c->buffer_queue_initial_size to %lu", initial_size); - c->buffer_queue_initial_size=initial_size; - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_BUFFER_QUEUE_INITIAL_SIZE value conversion failed (%s). using c->buffer_queue_initial_size default.", strerror(errno)); - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_BUFFER_QUEUE_INITIAL_SIZE is undefined. using c->buffer_queue_initial_size default"); - } - if ((env_str=getenv("TRIOS_NSSI_BUFFER_QUEUE_MAX_SIZE")) != NULL) { - errno=0; - uint32_t max_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(rpc_debug_level, "setting c->buffer_queue_max_size to %lu", max_size); - c->buffer_queue_max_size=max_size; - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_BUFFER_QUEUE_MAX_SIZE value conversion failed (%s). using c->buffer_queue_max_size default.", strerror(errno)); - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_BUFFER_QUEUE_MAX_SIZE is undefined. using c->buffer_queue_max_size default"); - } - if ((env_str=getenv("TRIOS_NSSI_BUFFER_QUEUE_CREATE_IF_EMPTY")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(rpc_debug_level, "setting c->buffer_queue_create_if_empty to TRUE"); - c->buffer_queue_create_if_empty=true; - } else { - log_debug(rpc_debug_level, "setting c->buffer_queue_create_if_empty to FALSE"); - c->buffer_queue_create_if_empty=false; - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_BUFFER_QUEUE_CREATE_IF_EMPTY is undefined. using c->buffer_queue_create_if_empty default"); - } - if ((env_str=getenv("TRIOS_NSSI_RDMA_BUFFER_QUEUE_INITIAL_SIZE")) != NULL) { - errno=0; - uint32_t initial_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(rpc_debug_level, "setting c->rdma_buffer_queue_initial_size to %lu", initial_size); - c->rdma_buffer_queue_initial_size=initial_size; - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_INITIAL_SIZE value conversion failed (%s). using c->rdma_buffer_queue_initial_size default.", strerror(errno)); - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_INITIAL_SIZE is undefined. using c->rdma_buffer_queue_initial_size default"); - } - if ((env_str=getenv("TRIOS_NSSI_RDMA_BUFFER_QUEUE_MAX_SIZE")) != NULL) { - errno=0; - uint32_t max_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(rpc_debug_level, "setting c->rdma_buffer_queue_max_size to %lu", max_size); - c->rdma_buffer_queue_max_size=max_size; - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_MAX_SIZE value conversion failed (%s). using c->rdma_buffer_queue_max_size default.", strerror(errno)); - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_MAX_SIZE is undefined. using c->rdma_buffer_queue_max_size default"); - } - if ((env_str=getenv("TRIOS_NSSI_RDMA_BUFFER_QUEUE_CREATE_IF_EMPTY")) != NULL) { - if ((!strcasecmp(env_str, "TRUE")) || - (!strcmp(env_str, "1"))) { - log_debug(rpc_debug_level, "setting c->rdma_buffer_queue_create_if_empty to TRUE"); - c->rdma_buffer_queue_create_if_empty=true; - } else { - log_debug(rpc_debug_level, "setting c->rdma_buffer_queue_create_if_empty to FALSE"); - c->rdma_buffer_queue_create_if_empty=false; - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_CREATE_IF_EMPTY is undefined. using c->rdma_buffer_queue_create_if_empty default"); - } - if ((env_str=getenv("TRIOS_NSSI_RDMA_BUFFER_QUEUE_BUFFER_SIZE")) != NULL) { - errno=0; - uint32_t max_size=strtoul(env_str, NULL, 0); - if (errno == 0) { - log_debug(rpc_debug_level, "setting c->rdma_buffer_queue_buffer_size to %lu", max_size); - c->rdma_buffer_queue_buffer_size=max_size; - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_BUFFER_SIZE value conversion failed (%s). using c->rdma_buffer_queue_buffer_size default.", strerror(errno)); - } - } else { - log_debug(rpc_debug_level, "TRIOS_NSSI_RDMA_BUFFER_QUEUE_BUFFER_SIZE is undefined. using c->rdma_buffer_queue_buffer_size default"); - } -} diff --git a/packages/trios/libraries/nessie/nssi/nssi_debug.c b/packages/trios/libraries/nessie/nssi/nssi_debug.c deleted file mode 100644 index d7df5e32917d..000000000000 --- a/packages/trios/libraries/nessie/nssi/nssi_debug.c +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file rpc_debug.c - * - * @brief Set the global debug level for the rpc mechanism. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 754 $. - * $Date: 2006-06-27 14:05:29 -0600 (Tue, 27 Jun 2006) $. - * - */ -#include "Trios_logger.h" -#include "Trios_nssi_debug.h" - -/* set to LOG_UNDEFINED -- log commands will use default level */ -log_level rpc_debug_level = LOG_UNDEFINED; diff --git a/packages/trios/libraries/nessie/nssi/nssi_opcodes.h b/packages/trios/libraries/nessie/nssi/nssi_opcodes.h deleted file mode 100644 index f5101d4c188f..000000000000 --- a/packages/trios/libraries/nessie/nssi/nssi_opcodes.h +++ /dev/null @@ -1,83 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file rpc_opcodes.h - * - * @brief Opcodes used by the all servers and clients. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 540 $ - * $Date: 2006-01-11 16:57:56 -0700 (Wed, 11 Jan 2006) $ - */ - -#ifndef _RPC_OPCODES_H_ -#define _RPC_OPCODES_H_ - -#include "Trios_nssi_types.h" - -#ifdef __cplusplus -extern "C" { -#endif - - enum nssi_rpc_opcodes { - /** @brief Null NSSI operation. */ - NSSI_OP_NULL = 1000, - - /** @brief Create a new object. */ - NSSI_OP_GET_SERVICE, - - /** @brief Kill an active service. */ - NSSI_OP_KILL_SERVICE, - - /** @brief Kill an active service. */ - NSSI_OP_TRACE_RESET - }; - - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/packages/trios/libraries/nessie/nssi/nssi_service_args.x b/packages/trios/libraries/nessie/nssi/nssi_service_args.x deleted file mode 100644 index 6fa0ef713865..000000000000 --- a/packages/trios/libraries/nessie/nssi/nssi_service_args.x +++ /dev/null @@ -1,100 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file nssi_service_args.x - * - * @brief XDR definitions for the argument structures for - * \ref generic service operations. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 969 $. - * $Date: 2006-08-28 15:40:44 -0600 (Mon, 28 Aug 2006) $. - * - */ - -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -/** - * @brief Arguments for the \ref nssi_create_container method that - * have to be passed to the authorization server. - */ -struct nssi_get_service_args { - - /** @brief The ID of the service (unused). */ - int id; -}; - -/** - * @brief Arguments for the \ref nssi_kill_service method that - * have to be passed to the authorization server. - */ -struct nssi_kill_service_args { - - /** @brief The signal to use when killing the service. */ - int sig; -}; - - -/** - * @brief Arguments for the \ref nssi_reset_tracing method. - */ -struct nssi_trace_reset_args { - - /** @brief The file type for the new tracing file. */ - int ftype; - - /** @brief The name of the new tracing file. */ - string fname; - - /** @brief Comma-separated list of traces to enable. */ - string enable<512>; -}; - diff --git a/packages/trios/libraries/nessie/nssi/nssi_trace.h b/packages/trios/libraries/nessie/nssi/nssi_trace.h deleted file mode 100644 index 159dc0964c5d..000000000000 --- a/packages/trios/libraries/nessie/nssi/nssi_trace.h +++ /dev/null @@ -1,88 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file rpc_trace.h - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 231 $ - * $Date: 2005-01-13 00:28:37 -0700 (Thu, 13 Jan 2005) $ - * - */ - -#ifndef _RPC_TRACE_H_ -#define _RPC_TRACE_H_ - -#include "nssi_opcodes.h" - -#ifdef __cplusplus -extern "C" { -#endif - - extern int ss_trace_level; - - enum RPC_TRACE_IDS { - TRACE_RPC_IDLE = NSSI_OP_NULL+1, - TRACE_RPC_PROC, - TRACE_RPC_DECODE, - TRACE_RPC_SENDRES, - TRACE_RPC_PUT_JOBWAIT, - TRACE_RPC_ADD_JOB, - TRACE_RPC_JOB_IDLE, - TRACE_RPC_UPDATE_JOBS - }; - -#define TRACE_RPC_COUNTER_GNAME "rpc-counter" -#define TRACE_RPC_INTERVAL_GNAME "rpc-interval" - -#if defined(__STDC__) || defined(__cplusplus) - -#else /* K&R C */ - -#endif /* K&R C */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/nessie/nssi/nssi_xdr.cpp b/packages/trios/libraries/nessie/nssi/nssi_xdr.cpp deleted file mode 100644 index 69fbdc4c37de..000000000000 --- a/packages/trios/libraries/nessie/nssi/nssi_xdr.cpp +++ /dev/null @@ -1,286 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file rpc_xdr.cc - * - * @brief XDR-encoding support for the - * \ref rpc_client_api "RPC Client API". - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 481 $ - * $Date: 2005-11-03 16:21:36 -0700 (Thu, 03 Nov 2005) $ - * - */ - -#include "Trios_config.h" -#include "Trios_nssi_types.h" - -#include - -#include -#ifdef HAVE_TRIOS_MALLOC_H -#include -#endif -#include -#include /* find malloc()/free() */ -#include /* find memset() */ - -#include "Trios_nssi_xdr.h" -#include "Trios_logger.h" -#include "Trios_threads.h" -#include "Trios_nssi_debug.h" - -#include "nssi_opcodes.h" -#include "nssi_service_args.h" - - -/** - * @brief XDR encodings for NSSI remote operations. - * - * The \ref xdr_encodings structure contains the - * xdr-encoding function pointers required for a remote - * NSSI function. - */ -struct xdr_encodings { - /** @brief A function pointer to encode/decode the arguments. */ - xdrproc_t encode_args; - /** @brief A function pointer to encode/decode the data. */ - xdrproc_t encode_data; - /** @brief A function pointer to encode/decode the result. */ - xdrproc_t encode_result; - - /** Default constructor */ - xdr_encodings() {} - - /** Constructor */ - xdr_encodings(xdrproc_t args, xdrproc_t data, xdrproc_t result) : - encode_args(args), encode_data(data), encode_result(result) {} - - /** Copy constructor */ - xdr_encodings(const struct xdr_encodings &encodings) - { - this->encode_args = encodings.encode_args; - this->encode_data = encodings.encode_data; - this->encode_result = encodings.encode_result; - } - - -}; - - -/** - * @brief A hashtable for XDR encodings of NSSI remote functions. - * - * The \ref encodings_ht hashtable is a thread-safe global - * data structure that holds encodings of registered NSSI functions. - * It is used by both clients and servers. - */ -static std::map encodings_map; -typedef std::map::iterator encodings_map_iterator_t; -static nthread_lock_t encodings_map_mutex; -static bool xdr_initialized = false; - - - -/* --------------------- Public functions ------------------- */ - -/** - * @brief Initialize the XDR encoding mechanism. - */ -int nssi_xdr_init() -{ - if (xdr_initialized) return NSSI_OK; - - /* check to see if logging is enabled */ - if (logger_not_initialized()) { - logger_set_file(stderr); - } - - nthread_lock_init(&encodings_map_mutex); - - xdr_initialized = true; - - return NSSI_OK; -} - -int register_service_encodings(void) -{ - int rc = NSSI_OK; - - /* get service */ - nssi_register_xdr_encoding(NSSI_OP_GET_SERVICE, - (xdrproc_t)NULL, - (xdrproc_t)NULL, - (xdrproc_t)&xdr_nssi_service); - - /* kill service */ - nssi_register_xdr_encoding(NSSI_OP_KILL_SERVICE, - (xdrproc_t)&xdr_nssi_kill_service_args, - (xdrproc_t)NULL, - (xdrproc_t)xdr_void); - - /* trace reset */ - nssi_register_xdr_encoding(NSSI_OP_TRACE_RESET, - (xdrproc_t)&xdr_nssi_trace_reset_args, - (xdrproc_t)NULL, - (xdrproc_t)xdr_void); - - return rc; -} - -/** - * @brief Finalize the XDR encoding mechanism. - */ -int nssi_xdr_fini() -{ - int rc = NSSI_OK; - - nssi_clear_xdr_encodings(); - - nthread_lock_fini(&encodings_map_mutex); - - xdr_initialized=false; - - return rc; -} - - -/** - * @brief Register xdr encodings for an opcode. - * - * The \ref nssi_register_xdr_encodings function registers - * the xdr encodings functions associated with a particular remote - * NSSI operation. - * - * @param opcode @input_type The opcode to lookup. - * @param encode_args @input_type The xdr function used to - * encode/decode the arguments. - * @param encode_data @input_type The xdr function used to - * encode/decode the data. - * @param encode_result @input_type The xdr function used to - * encode/decode the result. - */ -int nssi_register_xdr_encoding(int opcode, - xdrproc_t encode_args, - xdrproc_t encode_data, - xdrproc_t encode_result) -{ - int rc = NSSI_OK; - struct xdr_encodings encodings(encode_args, encode_data, encode_result); - - nssi_xdr_init(); - - log_debug(rpc_debug_level,"REGISTERING OPCODE=%d",opcode); - - /* insert structure into map */ - nthread_lock(&encodings_map_mutex); - encodings_map[(int)opcode] = encodings; /* should use copy constructor */ - nthread_unlock(&encodings_map_mutex); - - return rc; -} - - -/** - * @brief Lookup the encodings for an opcode. - * - * The \ref nssi_lookup_xdr_encodings function looks up - * the registered xdr encodings for an opcode. - * - * @param opcode @input_type The opcode to lookup. - * @param encode_args @output_type Points to the xdr function used to - * encode/decode the arguments. - * @param encode_data @output_type Points to the xdr function used to - * encode/decode the data. - * @param encode_result @output_type Points to the xdr function used to - * encode/decode the result. - * - * @return \ref NSSI_OK If successful. - * @return \ref NSSI_ENOENT If the encodings for the specified - * opcode do not exist. - */ -int nssi_lookup_xdr_encoding(int opcode, - xdrproc_t *encode_args, - xdrproc_t *encode_data, - xdrproc_t *encode_result) -{ - int rc = NSSI_OK; - encodings_map_iterator_t iter; - - nssi_xdr_init(); - - iter = encodings_map.find((int)opcode); - if ( iter == encodings_map.end()) { - log_warn(rpc_debug_level, "could not find encodings for opcode=%d",opcode); - return NSSI_ENOENT; - } - - *encode_args = iter->second.encode_args; - *encode_data = iter->second.encode_data; - *encode_result = iter->second.encode_result; - - return rc; -} - - -/** - * @brief Remove all encodings. - * - * The \ref nssi_clear_xdr_encodings function removes - * the xdr encodings functions associated with all remote - * NSSI operations. - * - */ -int nssi_clear_xdr_encodings() -{ - int rc = NSSI_OK; - - log_debug(rpc_debug_level,"DEREGISTERING ALL OPCODES"); - - /* insert structure into map */ - nthread_lock(&encodings_map_mutex); - encodings_map.clear(); - nthread_unlock(&encodings_map_mutex); - - return rc; -} diff --git a/packages/trios/libraries/nessie/server/Trios_nssi_server.h b/packages/trios/libraries/nessie/server/Trios_nssi_server.h deleted file mode 100644 index 8460d5c92957..000000000000 --- a/packages/trios/libraries/nessie/server/Trios_nssi_server.h +++ /dev/null @@ -1,475 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file nssi_server.h - * - * @brief Prototypes for the server-side methods for RPC. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - */ - -#ifndef _NSSI_RPC_SRVR_H_ -#define _NSSI_RPC_SRVR_H_ - -#include "Trios_nssi_rpc.h" - - - -/** - * @defgroup rpc_server_api Nessie Server API - * - * The server-side APIs for Nessie provide methods to identify - * supported call-back functions, register a service, transport - * data between client and server, and control the operations of - * an active service. - */ - -/* forward declaration */ -struct nssi_request; - -/* this is only for C++ code */ -#ifdef __cplusplus - -#include "Trios_nssi_TypeManip.h" - -/** - * @ingroup rpc_server_api - * @brief A macro for registering server callback functions. - * - * This macro constructs the service op structure with appropriate - * XDR-encoding functions, then adds the opcode to the list of - * supported ops for this service. - * - * @param id The opcode of the function to register. - * @param fptr The function pointer. - * @param arg_type Type for the function input arguments. - * @param res_type Type of the result. The stub will use nssi_send_result. - * - * @note We need a special version for C++ because sizeof(void) - * is not allowed in C++. See nssi_tests.h for - * the implementation of SizeOf. - */ -#define NSSI_REGISTER_SERVER_STUB(id, fptr, arg_type, res_type) \ - { \ - nssi_svc_op op; \ - op.opcode = id; \ - op.func = (nssi_rpc_proc)&fptr; \ - op.obj = (NssiRpc *)NULL; \ - op.sizeof_args = Nessie::SizeOf::value; \ - op.decode_args = (xdrproc_t)&xdr_ ## arg_type; \ - op.sizeof_res = Nessie::SizeOf::value; \ - op.encode_res = (xdrproc_t)&xdr_ ## res_type; \ - nssi_service_add_op( NULL, &op); \ - } - -/** - * @ingroup rpc_server_api - * @brief A macro for registering server callback function objects. - * - * This macro constructs the service op structure with appropriate - * XDR-encoding functions, then adds the opcode to the list of - * supported ops for this service. - * - * @param id The opcode of the function object to register. - * @param obj The function object pointer. - * @param arg_type Type for the function input arguments. - * @param res_type Type of the result. The stub will use nssi_send_result. - * - * @note We need a special version for C++ because sizeof(void) - * is not allowed in C++. See nssi_tests.h for - * the implementation of SizeOf. - */ -#define NSSI_REGISTER_SERVER_OBJ(id, func_obj, arg_type, res_type) \ - { \ - nssi_svc_op op; \ - op.opcode = id; \ - op.func = (nssi_rpc_proc)NULL; \ - op.obj = func_obj; \ - op.sizeof_args = Nessie::SizeOf::value; \ - op.decode_args = (xdrproc_t)&xdr_ ## arg_type; \ - op.sizeof_res = Nessie::SizeOf::value; \ - op.encode_res = (xdrproc_t)&xdr_ ## res_type; \ - nssi_service_add_op( NULL, &op); \ - } -#else - -/** - * @ingroup rpc_server_api - * @brief A macro for registering server callback functions. - * - * This macro constructs the service op structure with appropriate - * XDR-encoding functions, then adds the function to the list of - * supported ops for this service. - * - * @param id The opcode of the function to register. - * @param fptr The function pointer. - * @param arg_type Types for the function input arguments. - * @param res_type Type of the result. The stub will use nssi_send_result. - * - */ -#define NSSI_REGISTER_SERVER_STUB(id, fptr, arg_type, res_type) \ - { \ - nssi_svc_op op; \ - op.opcode = id; \ - op.func = (nssi_rpc_proc)&fptr; \ - op.obj = (void *)NULL; \ - op.sizeof_args = sizeof(arg_type); \ - op.decode_args = (xdrproc_t)&xdr_ ## arg_type; \ - op.sizeof_res = sizeof(res_type); \ - op.encode_res = (xdrproc_t)&xdr_ ## res_type; \ - nssi_service_add_op( NULL, &op); \ - } -#endif - - - - -#ifdef __cplusplus -extern "C" { -#endif - - - /** - * @ingroup rpc_server_api - * @brief Definition of a function pointer for RPC services. - * - * Each server-side stub uses the following parameters: - * - \ref nssi_remote_pid *caller : the remote PID of the calling process - * - void *args : arguments from the client - * - const nssi_rma *data_addr : remote address for get/put of raw data on the client - * - const nssi_rma *result_addr: remote address for result buffer (for \ref nssi_send_result) - */ - typedef int (*nssi_rpc_proc) ( - const unsigned long request_id, - const NNTI_peer_t *, - const void *, - const NNTI_buffer_t *, - const NNTI_buffer_t *); - - -#ifdef __cplusplus - class NssiRpc { - public: - virtual int doRPC( - const int opcode, - const unsigned long request_id, - const NNTI_peer_t *caller, - const void *void_args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) = 0; -// virtual void registerRPC() = 0; - }; -#else - /* In C code, we need a substitute for class NssiRpc. */ - typedef uint8_t NssiRpc; -#endif - - - - /** - * @brief A structure associated with an operation made available - * by a registered service. - * - * The \b nssi_svc_op structure includes all fields required to - * spawn call a local method when an RPC request arrives. - */ - typedef struct nssi_svc_op { - /** @brief The operation code. */ - int opcode; - - /** @brief Function pointer for the server-side function. */ - nssi_rpc_proc func; - - /** @brief Function pointer for the server-side function. */ - NssiRpc *obj; - - /** @brief Size of the arguments structure. */ - uint32_t sizeof_args; - - /** @brief A function to decode the arguments when a request arrives. */ - xdrproc_t decode_args; - - /** @brief Size of the result structure. */ - uint32_t sizeof_res; - - /** @brief A function to encode the result after servicing the request. */ - xdrproc_t encode_res; - } nssi_svc_op; - - - /** - * @brief Structure for a list of RPC services. - */ - typedef struct nssi_svc_op_list { - - /** @brief The service for this entry. */ - nssi_svc_op svc_op; - - /** @brief The remaining list of services. */ - struct nssi_svc_op_list *next; - } nssi_svc_op_list; - - typedef struct rpc_request { - nssi_service *svc; - NNTI_peer_t caller; - char *req_buf; - nssi_size short_req_len; - int id; - double arrival_time; - } nssi_svc_rpc_request; - -#if defined(__STDC__) || defined(__cplusplus) - - /** - * @ingroup rpc_server_api - * @brief Create a daemon process. - */ - extern void nssi_daemonize(); - - /** - * @ingroup rpc_server_api - * @brief An abstract method to get data from a remote memory descriptor. - * - * The server stub uses this function to get or put data to a - * client memory descriptor. - * - * @param caller Remote process ID of the destination node. - * @param buf the buffer for the data. - * @param len the maximum length of the buffer. - * @param data_addr the remote memory descriptor. - * - * @todo Get rid of the caller parameter. - */ - extern int nssi_get_data( - const NNTI_peer_t *caller, - void *buf, - const int len, - const NNTI_buffer_t *data_addr); - - /** - * @ingroup rpc_server_api - * @brief An abstract method to put data into a remote memory descriptor. - * - * The server stub uses this function to put data to a - * client memory descriptor. - * - * @param caller Remote process ID of the destination. - * @param buf Buffer for the data. - * @param len The amount of data to send. - * @param data_addr The remote memory descriptor. - * @param timeout Time to wait for response from the destination. - * - * @todo Get rid of the caller parameter. - */ - extern int nssi_put_data( - const NNTI_peer_t *caller, - const void *buf, - const int len, - const NNTI_buffer_t *data_addr, - const long timeout); - - /** - * @ingroup rpc_server_api - * @brief Send result back to the client. - * - * This function allows the service library to send a result to - * a remote memory descriptor. The result will be packaged as - * an \ref nssi_result_header, then PUT on the memory descriptor of - * the remote address. - * - * @param caller The remote process ID of the caller. - * @param request_id ID of the request associated with this result. - * @param rc The return code for the server function. - * @param result Pointer to the result data structure (NULL if not used) - * @param result_addr The remote memory address of the client (where - * to PUT the result) - * - * @returns \ref NSSI_OK if successful. - */ - extern int nssi_send_result( - const NNTI_peer_t *caller, - unsigned long request_id, - const int rc, - void *result, - const NNTI_buffer_t *result_addr); - - - /** - * @ingroup rpc_server_api - * @brief Initialize an RPC server. - * - * @ingroup rpc_server_api - * - * This method initializes the portals library and allocates space - * for incoming requests. - * - * @param match_bits the portals match bits - * @param short_req_len the length of a portals short request queue - * @param service the service descriptor to register (to register for clients). - */ - extern int nssi_service_init( - const nssi_rpc_transport rpc_transport, - const int short_req_len, - nssi_service *service); - - - /** - * @ingroup rpc_server_api - * @brief Cleanly abort the RPC server. - * - * The abort function kills a running service by sending a - * SIGINT signal to the running process. If the service - * has already started, the signal is caught by the - * signal handler. - */ - extern void nssi_service_abort(); - - /** - * @ingroup rpc_server_api - * @brief Returns true if the service needs to shut down. - */ - extern int nssi_exit_now(); - - - /** - * @ingroup rpc_server_api - * @brief Process an encoded RPC request. - * - * This function processes an encoded RPC request by decoding the - * header, then calling a registered callback based on the opcode - * sent in the header. - */ - extern int nssi_process_rpc_request( - nssi_svc_rpc_request *req); - - - /** - * @ingroup rpc_server_api - * @brief Start an RPC service. - * - * The \b nssi_service_start implements a loop that waits for incoming - * RPC requests, then calls the nssi_process_rpc_request function to - * process those requests. - * - * @param service The service descriptor. - */ - extern int nssi_service_start( - nssi_service *service); - - /** - * This function is essentially the same as nssi_start, but it allows - * the caller to pass in a different function to process encoded requests. - * This is useful, for example to implement a multithreaded service. The - * code could call an "enqueue_rpc_request" function and have a separate - * thread that pops requests off the queue and processes them. - */ - extern int nssi_service_start_wfn( - nssi_service *svc, - int (*process_req)(nssi_svc_rpc_request *req)); - - - /* - * Private function for registering new callbacks for this service. - */ - extern int nssi_service_add_op( - const nssi_service *svc, - const nssi_svc_op *ops); - - - /** - * @ingroup rpc_server_api - * @brief Close down an active service. - * - * Shuts down the service and releases any associated resources. - * - * @param service The service descriptor. - */ - extern int nssi_service_fini( - const nssi_service *service); - - /** - * @ingroup rpc_server_api - * @brief Register an RPC service. - * - * This method creates a named RPC service on the specified - * registry server. Along with the name of the service, the - * server has to specify where (in the form of an \ref nssi_rma) the - * client should "put" requests. - * - * @param registry_id Process ID of the registry server. - * @param name Name of the service. - * @param service The service description to associate with the name. - * @param req The request handle (used to test for completion). - * - * @note This function is not yet implemented. - */ - extern int nssi_register_service( - const NNTI_peer_t registry_id, - const char *name, - const nssi_service *service, - nssi_request *req); - - /** - * @brief Return the age, in seconds, of the specified request. - * - * For active requests, the age is the current time minus the arrival time - * of the request. For expired request (if we support those), it is the - * completion-arrival time. - */ - extern double nssi_get_request_age(const NNTI_peer_t *caller, const int req_id); - - -#else /* K&R C */ -#endif - - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/packages/trios/libraries/nessie/server/nssi_server.cpp b/packages/trios/libraries/nessie/server/nssi_server.cpp deleted file mode 100644 index dbe930f1218d..000000000000 --- a/packages/trios/libraries/nessie/server/nssi_server.cpp +++ /dev/null @@ -1,2052 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file rpc_server.c - * - * @brief Implementation of the \ref rpc_server_api "RPC server API". - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1654 $ - * $Date: 2007-12-11 22:57:34 -0700 (Tue, 11 Dec 2007) $ - * - */ - - -#include "Trios_config.h" -#include "Trios_nssi_server.h" - -#include "Trios_nssi_types.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" -#include "Trios_nssi_xdr.h" -#include "Trios_nnti.h" - -#include -#ifdef HAVE_TRIOS_MALLOC_H -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_signal.h" -#include "Trios_trace.h" -#include "Trios_threads.h" -#include "Trios_nssi_rpc.h" -#include "buffer_queue.h" -#include "Trios_nssi_debug.h" - -#include "nssi_opcodes.h" -#include "nssi_trace.h" -#include "nssi_service_args.h" - - - -extern NNTI_transport_t transports[NSSI_RPC_COUNT]; -extern nssi_config_t nssi_config; - -extern trios_buffer_queue_t send_bq; -extern trios_buffer_queue_t recv_bq; -extern trios_buffer_queue_t rdma_target_bq; -extern trios_buffer_queue_t rdma_get_bq; -extern trios_buffer_queue_t rdma_put_bq; - - -int rpc_get_service( - const unsigned long request_id, - const NNTI_peer_t *caller, - const void *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr); - -int rpc_kill_service( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nssi_kill_service_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr); - -int rpc_trace_reset( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nssi_trace_reset_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr); - -typedef void (*progress_callback)(bool is_idle); - - -static bool time_to_die = false; - -static nssi_service local_service; - -#undef USE_THREADED_SERVERS - - -static std::map supported_ops; -typedef std::map::iterator supported_ops_iterator_t; -static nthread_lock_t supported_ops_mutex; - - -unsigned long max_mem_allowed=0; - -static int trace_counter_gid; -static int trace_interval_gid; - - -#ifdef GNI_PERF -#include -gemini_state_t gni_state; -#endif - -/* Need a struct to encapsulate a IB connection addr/port pair. - * STL does not like arrays in template defs. - */ -struct caller_reqid { - NNTI_peer_t caller; - unsigned long reqid; - - caller_reqid(const NNTI_peer_t *c, const unsigned long r) { - caller=*c; - reqid=r; - } -}; -/* Need a comparison operator to pass into the conn_map. - */ -struct caller_reqid_lt -{ - bool operator()(const struct caller_reqid &cr1, const struct caller_reqid &cr2) const - { - if (strcmp(cr1.caller.url, cr2.caller.url)<0) return TRUE; - if ((strcmp(cr1.caller.url, cr2.caller.url)==0) && - (cr1.reqid < cr2.reqid)) return TRUE; - - return FALSE; - } -}; - -typedef struct { - int opcode; - unsigned long request_id; - uint64_t start_time; - double arrival_time; - - NNTI_buffer_t *data_hdl; - NNTI_buffer_t shadow_data; - NNTI_buffer_t *shadow_data_hdl; - - int8_t is_responseless; -} request_args_t; - -static std::map request_args_map; -typedef std::map::iterator request_args_map_iterator_t; -typedef std::pair request_args_map_pair_t; - -static nthread_lock_t request_args_map_mutex; - -typedef struct { - const NNTI_buffer_t *data_buffer; // this is the original client data buffer - const NNTI_buffer_t *shadow_buffer; // this is the shadow buffer on the server - bool data_was_got; - bool data_was_put; -} shadow_buffer_entry; -static std::map shadow_buffer_map; -typedef std::map::iterator shadow_buffer_map_iterator_t; -static nthread_lock_t shadow_buffer_mutex; - -static log_level shadow_debug_level = LOG_UNDEFINED; - - - -//static void print_raw_buf(void *buf, uint32_t size) -//{ -// if (logging_debug(rpc_debug_level)) { -// FILE* f=logger_get_file(); -// uint64_t print_limit=(size<96) ? size : 96; -// fprintf(f, "\nbuf (%p)\n", buf); -// fflush(f); -// if (buf != NULL) { -// uint64_t l=0; -// for (l=0;l ", (uint64_t)buf, l); -// fprintf(f, "%02hhX", ((char *)buf)[l]); -// } -// fprintf(f, "\n"); -// } -// } -//} - - - -void request_args_add(const NNTI_peer_t *caller, const unsigned long reqid, request_args_t *request_args) -{ - caller_reqid cr(caller, reqid); - - log_debug(rpc_debug_level, "enter - adding caller(%s) reqid(%lu)", - caller->url, reqid); - - if (nthread_lock(&request_args_map_mutex)) log_warn(rpc_debug_level, "failed to get thread lock"); - request_args_map[cr] = request_args; - log_debug(rpc_debug_level, "request_args_map.size() == %lu", request_args_map.size()); - nthread_unlock(&request_args_map_mutex); - log_debug(rpc_debug_level, "end"); -} - -request_args_t *request_args_get(const NNTI_peer_t *caller, const unsigned long reqid) -{ - caller_reqid cr(caller, reqid); - - log_debug(rpc_debug_level, "enter - looking for caller(%s) reqid(%lu)", - caller->url, reqid); - - if (nthread_lock(&request_args_map_mutex)) log_warn(rpc_debug_level, "failed to get thread lock"); - request_args_map_iterator_t iter=request_args_map.find(cr); - request_args_t *request_args=iter->second; - log_debug(rpc_debug_level, "request_args_map.size() == %lu", request_args_map.size()); - nthread_unlock(&request_args_map_mutex); - - log_debug(rpc_debug_level, "end"); - - return(request_args); -} - -void request_args_del(const NNTI_peer_t *caller, const unsigned long reqid) -{ - caller_reqid cr(caller, reqid); - request_args_t *request_args=NULL; - - log_debug(rpc_debug_level, "enter - deleting caller(%s) reqid(%lu)", - caller->url, reqid); - - if (nthread_lock(&request_args_map_mutex)) log_warn(rpc_debug_level, "failed to get thread lock"); - request_args_map_iterator_t iter=request_args_map.find(cr); - // it's OK if the caller/reqid is not found. could have been an error before request args were added. - if (iter != request_args_map.end()) { - request_args=iter->second; - request_args_map.erase(iter); - log_debug(rpc_debug_level, "request_args_map.size() == %lu", request_args_map.size()); - if (request_args != NULL) { - free(request_args); - } - } - nthread_unlock(&request_args_map_mutex); - - log_debug(rpc_debug_level, "end"); -} - - -static void print_shadow_buffer_map() -{ - log_level debug_level=shadow_debug_level; - - if (!logging_debug(debug_level)) { - return; - } - - shadow_buffer_map_iterator_t i; - if (nthread_lock(&shadow_buffer_mutex)) log_warn(shadow_debug_level, "failed to get thread lock"); - for (i=shadow_buffer_map.begin(); i != shadow_buffer_map.end(); i++) { - log_debug(debug_level, "shadow_buffer_map key=%p sbe=%p", i->first, i->second); - } - nthread_unlock(&shadow_buffer_mutex); -} -static NNTI_result_t insert_shadow_buffer(shadow_buffer_entry *sbe) -{ - NNTI_result_t rc=NNTI_OK; - - if (nthread_lock(&shadow_buffer_mutex)) log_warn(shadow_debug_level, "failed to get thread lock"); - - log_debug(shadow_debug_level, "adding shadow buffer (sbe=%p)", sbe); - - assert(shadow_buffer_map.find(sbe->shadow_buffer) == shadow_buffer_map.end()); - shadow_buffer_map[sbe->shadow_buffer] = sbe; - - log_debug(shadow_debug_level, "added shadow buffer (sbe=%p)", sbe); - - nthread_unlock(&shadow_buffer_mutex); - - return(rc); - -} -static shadow_buffer_entry *get_shadow_buffer(const NNTI_buffer_t *sbuf) -{ - shadow_buffer_entry *sbe=NULL; - - if (nthread_lock(&shadow_buffer_mutex)) log_warn(shadow_debug_level, "failed to get thread lock"); - - log_debug(shadow_debug_level, "looking for shadow buffer (sbuf=%p)", sbuf); - - if (shadow_buffer_map.find(sbuf) != shadow_buffer_map.end()) { - sbe = shadow_buffer_map[sbuf]; - } - - log_debug(shadow_debug_level, "found shadow buffer entry (sbe=%p)", sbe); - - nthread_unlock(&shadow_buffer_mutex); - - if (sbe != NULL) { - log_debug(shadow_debug_level, "shadow buffer entry found (sbe=%p)", sbe); - return sbe; - } - - log_debug(shadow_debug_level, "shadow buffer entry NOT found"); - - print_shadow_buffer_map(); - - return(NULL); -} -static shadow_buffer_entry *del_shadow_buffer(NNTI_buffer_t *victim) -{ - shadow_buffer_entry *sbe=NULL; - - log_level debug_level = shadow_debug_level; - - if (nthread_lock(&shadow_buffer_mutex)) log_warn(shadow_debug_level, "failed to get thread lock"); - - log_debug(debug_level, "deleting shadow buffer (victim=%p)", victim); - - if (shadow_buffer_map.find(victim) != shadow_buffer_map.end()) { - sbe = shadow_buffer_map[victim]; - } - - if (sbe != NULL) { - log_debug(shadow_debug_level, "shadow buffer entry found and deleted (sbe=%p ; victim=%p)", sbe, victim); - shadow_buffer_map.erase(victim); - } else { - log_debug(debug_level, "shadow buffer entry NOT found"); - } - - log_debug(debug_level, "deleted (sbe=%p ; victim=%p)", sbe, victim); - - nthread_unlock(&shadow_buffer_mutex); - - print_shadow_buffer_map(); - - return(sbe); -} - - - -/* ----------- Implementation of core services ----------- */ - -/** - * @brief Return the service description of this service. - * - * @param caller @input_type the client's PID - * @param args @input_type arguments needed to verify the container - * @param data_addr @input_type address at which the bulk data can be found - * @param res_addr @input_type address of remote result buffer - * - * @returns The service descriptor for this service. - */ -int rpc_get_service( - const unsigned long request_id, - const NNTI_peer_t *caller, - const void *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NSSI_OK; - - /* copy the service description into the result */ - log_debug(rpc_debug_level, "entered get service"); - - /* send the local_service descriptor to the client */ - rc = nssi_send_result(caller, request_id, rc, &local_service, res_addr); - if (rc != NSSI_OK) { - log_warn(rpc_debug_level, "Could not send service description to client"); - } - - return rc; -} - - -/** - * @brief Schedule this service to be killed. - * - * @param caller @input_type the client's PID - * @param args @input_type arguments needed to verify the container - * @param data_addr @input_type address at which the bulk data can be found - * @param res_addr @input_type address of remote result buffer - * - * @returns The service descriptor for this service. - */ -int rpc_kill_service( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nssi_kill_service_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NSSI_OK; - - /* copy the service description into the result */ - log_debug(rpc_debug_level, "killing service"); - - /* set the "exit_now" flag. */ - /*nssi_abort();*/ - - switch (args->sig) { - - case 0: - /* Graceful exit. When threads come out of timeout, they will exit */ - time_to_die = true; - log_debug(rpc_debug_level, "Graceful Abort"); - //trios_abort(); /* sets the exit_now flag */ - break; - - default: - /* Force exit. When things just must die! */ - log_debug(LOG_ALL, "Forced Abort"); - trios_abort(); - exit(args->sig); - break; - - } - - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - if (rc != NSSI_OK) { - log_warn(rpc_debug_level, "Unable to send result from kill"); - } - - return rc; -} - - -/** - * @brief Reset the tracing library. - * - * If the service is currently using the tracing - * library, this operation will reset the tracing - * library... forcing a flush of the previous file - * and creating new file for the trace data. All - * counts and timers are reset to 0. - * - * @param caller @input_type the client's PID - * @param args @input_type arguments needed to verify the container - * @param data_addr @input_type address at which the bulk data can be found - * @param result @output_type no result - * - * @returns The service descriptor for this service. - */ -int rpc_trace_reset( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nssi_trace_reset_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NSSI_OK; - - char *fname; - const int ftype = args->ftype; - char *enable; - - if ((!args->fname) || (strcmp(args->fname,"")==0)) { - fname = NULL; - } - else { - fname = (char *)args->fname; - } - - if ((!args->enable) || (strcmp(args->enable,"")==0)) { - enable = NULL; - } - else { - enable = (char *)args->enable; - } - - - /* copy the service description into the result */ - log_debug(rpc_debug_level, "reset tracing(%s, %d)", - fname, ftype); - - /* set the "exit_now" flag. */ - /*trace_reset(enable_flag, fname, ftype);*/ - //trace_fini(); - //trace_init(fname, ftype); - - trace_reset(fname, ftype, enable); - - /* send result back to client */ - nssi_send_result(caller, request_id, rc, NULL, res_addr); - return rc; -} - - -/* ----------- Implementation of the NSSI messaging -------- */ - -/** - * @brief Fetch or extract the operation arguments. - * - * If the args were small enough to fit into the short - * request buffer, we extract the args directly from - * the buffer. Otherwise, we GET the arguments from - * a remote memory descriptor on the client. - * - * @param encoded_buf The encoded short request buffer. - * @param header The request header. - * @param decode_args The XDR function that decodes the arguments. - * @param args Where to place the decoded arguments. - */ -static int fetch_args( - NNTI_peer_t *caller, - nssi_request_header *header, - xdrproc_t xdr_decode_args, - void *args) -{ - int rc; /* return code from non-NSSI methods */ - XDR xdrs; - - trios_declare_timer(call_time); - - /* pointer to the decoded buffer for arguments */ - char *buf=NULL; - NNTI_buffer_t encoded_args_hdl; - NNTI_work_request_t encoded_args_wr; - NNTI_status_t status; - nssi_size encoded_args_size = NNTI_BUFFER_SIZE(&header->args_addr); - - /* allocate the decoded buffer */ - rc=NNTI_alloc( - &transports[caller->peer.transport_id], - encoded_args_size, - 1, - NNTI_GET_DST, - &encoded_args_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering long args: %s", - nnti_err_str(rc)); - goto cleanup; - } - buf=NNTI_BUFFER_C_POINTER(&encoded_args_hdl); - - assert(header->fetch_args); - - log_debug(rpc_debug_level, - "get args from client"); - - /* fetch the buffer from the client */ - trios_start_timer(call_time); - rc=NNTI_get( - &header->args_addr, - 0, - encoded_args_size, - &encoded_args_hdl, - 0, - &encoded_args_wr); - trios_stop_timer("NNTI_get - long args", call_time); - if (rc != NNTI_OK) { - log_fatal(rpc_debug_level, - "could not get long args from client"); - goto cleanup; - } - trios_start_timer(call_time); - rc=NNTI_wait( - &encoded_args_wr, - -1, - &status); - trios_stop_timer("NNTI_wait - long args", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for long args: %s", - nnti_err_str(rc)); - goto cleanup; - } - - /* decode the arguments */ - log_debug(rpc_debug_level,"decoding args, size=%d", - encoded_args_size); - - /* create an xdr memory stream for the decoded args */ - xdrmem_create( - &xdrs, - NNTI_BUFFER_C_POINTER(&encoded_args_hdl), - NNTI_BUFFER_SIZE(&encoded_args_hdl), - XDR_DECODE); - /* decode -- will allocate memory if necessary */ - trios_start_timer(call_time); - if (!xdr_decode_args(&xdrs, args)) { - log_fatal(rpc_debug_level,"could not decode args"); - fprint_NNTI_status(logger_get_file(), "status", "FATAL", &status); -// print_raw_buf(NNTI_BUFFER_C_POINTER(&encoded_args_hdl), -// NNTI_BUFFER_SIZE(&encoded_args_hdl)); -// fflush(logger_get_file()); - rc = NSSI_EDECODE; - goto cleanup; - } - trios_stop_timer("xdr_decode_args - decode", call_time); - -cleanup: - /* if we had to fetch the args, we need to free the buffer */ - if (buf) { - int cleanup_rc; - cleanup_rc=NNTI_free(&encoded_args_hdl); - if (cleanup_rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering long args: %s", - nnti_err_str(cleanup_rc)); - } - } - - return rc; -} - -/** - * @brief Send the result back to the client. - * - * If the result is small enough to fit inside a small result - * buffer, the results are sent in one message transfer. If the - * results are too large, we tell the client to fetch the result - * (by setting the fetch_result flag of the result header to true) - * send the result header, then wait for the client to fetch the - * result. - * - * @param dest @input Where to send the encoded result. - * @param xdr_encode_result @input function used to encode result. - * @param return_code @input The return code of the function. - * @param result @input the result of the function. - */ -static int send_result(const NNTI_peer_t *caller, - const unsigned long request_id, - const NNTI_buffer_t *dest_addr, - xdrproc_t xdr_encode_result, - const int return_code, - void *result) -{ - static uint32_t res_counter = 1; - - trios_declare_timer(call_time); - - int rc; /* return code for non-NSSI methods */ - - uint32_t hdr_size; - uint32_t res_size; - uint32_t res_buf_size; - uint32_t remaining; - uint32_t valid_bytes; - char *buf=NULL; - NNTI_buffer_t short_res; - NNTI_buffer_t *short_res_hdl=&short_res; - NNTI_work_request_t short_res_wr; - NNTI_buffer_t long_res_hdl; - NNTI_status_t wait_status; - nssi_result_header header; - - NNTI_buffer_t long_res_ack_hdl; - NNTI_work_request_t long_res_ack_wr; - NNTI_status_t long_res_ack_status; - - - request_args_t *args=request_args_get(caller, request_id); - int opcode=args->opcode; - - /* xdrs for the header and the result. */ - XDR hdr_xdrs, res_xdrs; - - /* initialize the result header */ - memset(&header, 0, sizeof(nssi_result_header)); - - /* --- HANDLE ERROR CASE --- */ - - /* If a function returns an error, we return the error code, but - * no result. */ - if (return_code != NSSI_OK) { - - /* treat as if there is no result to return */ - xdr_encode_result = (xdrproc_t)&xdr_void; - } - - /* --- CALCULATE SIZES --- */ - - /* Calculate size of the encoded header */ - hdr_size = xdr_sizeof((xdrproc_t)&xdr_nssi_result_header, &header); - - /* Calculate size of the encoded result */ - if (result == NULL) { - res_size = 0; - } else { - res_size = xdr_sizeof(xdr_encode_result, result); - } - - /* Extract the size of the client-side buffer for the result */ - res_buf_size = NNTI_BUFFER_SIZE(dest_addr); - - /* Calculate space left in the short result buffer */ - remaining = res_buf_size - hdr_size; - - /* allocated an xdr memory stream for the short result buffer */ - if (res_buf_size <= 0) { - log_error(rpc_debug_level, "********** res_buf_size (%lu) <= 0", res_buf_size); - fprint_NNTI_buffer(logger_get_file(), "dest_addr", "ERROR res_buf_size<=0 %", dest_addr); - } - assert(res_buf_size > 0); - - if (nssi_config.use_buffer_queue) { - short_res_hdl=trios_buffer_queue_pop(&send_bq); - assert(short_res_hdl); - } else { - rc=NNTI_alloc( - &transports[caller->peer.transport_id], - res_buf_size, - 1, - NNTI_SEND_SRC, - short_res_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering short result: %s", - nnti_err_str(rc)); - } - buf=NNTI_BUFFER_C_POINTER(short_res_hdl); - memset(buf, 0, res_buf_size); // address valgrind uninitialized error - } - - xdrmem_create( - &hdr_xdrs, - NNTI_BUFFER_C_POINTER(short_res_hdl), - NNTI_BUFFER_SIZE(short_res_hdl), - XDR_ENCODE); - - /* If the result fits in the short result buffer, send it with the header */ - if (res_size < remaining) { - - log_debug(rpc_debug_level, "sending short_result %lu, " - "result buffer size = %d, header size = %d, available space = %d, result_size = %d", - request_id, res_buf_size, hdr_size, remaining, res_size); - - /* client needs this information from the header */ - header.fetch_result = FALSE; - header.id = request_id; - header.opcode = opcode; - header.result_size = res_size; - header.rc = return_code; - - /* encode the header */ - log_debug(rpc_debug_level, "encode result header"); - trios_start_timer(call_time); - if (!xdr_nssi_result_header(&hdr_xdrs, &header)) { - log_fatal(rpc_debug_level, "failed to encode the result header"); - return NSSI_EENCODE; - } - trios_stop_timer("xdr_nssi_result_header - encode", call_time); - - /* encode the result in the header */ - log_debug(rpc_debug_level, "encode result data"); - if (result != NULL) { - trios_start_timer(call_time); - if (!xdr_encode_result(&hdr_xdrs, result)) { - log_fatal(rpc_debug_level, "failed to encode the result"); - return NSSI_EENCODE; - } - trios_stop_timer("xdr_encode_result - encode", call_time); - } - } - - /* if result does not fit, client has to fetch result */ - else { - - res_counter++; - - log_debug(rpc_debug_level, "sending long result %lu, " - "available space = %d, result_size = %d", request_id, remaining, - res_size); - - /* allocate memory for the result - * structure keeps track of the buffer so it can free - * the memory later. */ - rc=NNTI_alloc( - &transports[caller->peer.transport_id], - res_size, - 1, - NNTI_GET_SRC, - &long_res_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering long result: %s", - nnti_err_str(rc)); - } - buf=NNTI_BUFFER_C_POINTER(&long_res_hdl); - - header.result_addr=long_res_hdl; - - log_debug(rpc_debug_level, "allocated long_res_buf(%lu) req_id(%lu)", buf, request_id); - - trios_start_timer(call_time); - rc=NNTI_alloc( - &transports[caller->peer.transport_id], - sizeof(int8_t), - 1, - NNTI_RECV_DST, - &long_res_ack_hdl); - trios_stop_timer("NNTI_register_memory - long result ack", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering long result ack: %s", - nnti_err_str(rc)); - } - - header.result_ack_addr=long_res_ack_hdl; - - /* we want the client to fetch the result */ - /* client needs this information from the header */ - header.fetch_result = TRUE; - header.id = request_id; - header.opcode = opcode; - header.result_size = res_size; - header.rc = return_code; - - /* create a xdr memory stream for the encoded args buffer */ - xdrmem_create( - &res_xdrs, - NNTI_BUFFER_C_POINTER(&long_res_hdl), - NNTI_BUFFER_SIZE(&long_res_hdl), - XDR_ENCODE); - - /* encode the header */ - log_debug(rpc_debug_level, "encode result %lu header", - request_id); - trios_start_timer(call_time); - if (!xdr_nssi_result_header(&hdr_xdrs, &header)) { - log_fatal(rpc_debug_level, "failed to encode the result header"); - rc = NSSI_EENCODE; - goto cleanup; - } - trios_stop_timer("xdr_nssi_result_header - encode", call_time); - - /* encode the result */ - log_debug(rpc_debug_level, "encode result %lu data", request_id); - trios_start_timer(call_time); - if (!xdr_encode_result(&res_xdrs, result)) { - log_fatal(rpc_debug_level, "failed to encode the result"); - rc = NSSI_EENCODE; - goto cleanup; - } - trios_stop_timer("xdr_encode_result - encode", call_time); - -// if (logging_debug(rpc_debug_level)) { -// u_int64_t print_limit=(long_res_buf->msg_size<90) ? long_res_buf->msg_size : 90; -// for (int l=0;l ", (uint64_t)long_res_buf->msg, request_id, l); -// fprintf(stdout, "%02hhX", ((char *)long_res_buf->msg)[l]); -// } -// fprintf(stdout, "\n"); -// } - } - - if (logging_debug(rpc_debug_level)) { - fprint_nssi_result_header(logger_get_file(), "header", "nssi_result_header", &header); - } - - /* send the short result to the client */ - valid_bytes = hdr_size; - - if (!header.fetch_result) - valid_bytes += res_size; - - if (logging_debug(rpc_debug_level)) { - log_debug(rpc_debug_level, "send short result %lu " - "(in xdr bytes: len=%d bytes: encoded_header=%d bytes, res=%d bytes)", - request_id, valid_bytes, hdr_size, res_size); - - fprint_NNTI_buffer(logger_get_file(), "nssi_server: 768 -- dest_addr", "%", dest_addr); - - fprint_NNTI_buffer(logger_get_file(), "nssi_server: 768 -- short_res_hdl", "%", short_res_hdl); - - fprint_NNTI_peer(logger_get_file(), "nssi_server: 772 -- caller", "%", caller); - } - - - /* TODO: Handle the timeout case. This probably means the client died */ - trios_start_timer(call_time); - rc=NNTI_send( - caller, - short_res_hdl, - dest_addr, - &short_res_wr); - trios_stop_timer("NNTI_send - short result", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed sending short result: %s", - nnti_err_str(rc)); - } - trios_start_timer(call_time); - rc=NNTI_wait( - &short_res_wr, - -1, - &wait_status); - trios_stop_timer("NNTI_wait - short result", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for short result: %s", - nnti_err_str(rc)); - goto cleanup; - } - - - /* if the client has to fetch the results, we need to wait for - * the GET to complete */ - if (header.fetch_result) { - log_debug(rpc_debug_level, "waiting for client to " - "ACK request %lu", request_id); - - trios_start_timer(call_time); - NNTI_create_work_request( - &long_res_ack_hdl, - &long_res_ack_wr); - rc=NNTI_wait( - &long_res_ack_wr, - -1, - &long_res_ack_status); - NNTI_destroy_work_request( - &long_res_ack_wr); - trios_stop_timer("NNTI_wait - long result ack", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for client to send long result ack: %s", - nnti_err_str(rc)); - } - } - -cleanup: - if (header.fetch_result) { - rc=NNTI_free(&long_res_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering long result: %s", - nnti_err_str(rc)); - } - - rc=NNTI_free(&long_res_ack_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering long result ack: %s", - nnti_err_str(rc)); - } - } - - if (nssi_config.use_buffer_queue) { - trios_buffer_queue_push(&send_bq, short_res_hdl); - } else { - rc=NNTI_free(short_res_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering short result: %s", - nnti_err_str(rc)); - } - } - - log_debug(rpc_debug_level, "result %lu sent", request_id); - - return rc; -} - -/** - * Lookup an opcode in the list of supported ops. - */ -static int lookup_service_op( - const int opcode, - nssi_svc_op *result_op) -{ - int rc = NSSI_OK; - - log_debug(rpc_debug_level, "enter (opcode=%d)", opcode); - - log_debug(rpc_debug_level, "locking ops mutex"); - if (nthread_lock(&supported_ops_mutex)) log_warn(rpc_debug_level, "failed to get thread lock"); - log_debug(rpc_debug_level, "locked ops mutex"); - if (supported_ops.find(opcode) == supported_ops.end()) { - rc = NSSI_ENOENT; - } else { - /* should use the copy constructor */ - *result_op = supported_ops[(int)opcode]; - } - log_debug(rpc_debug_level, "unlocking ops mutex"); - nthread_unlock(&supported_ops_mutex); - log_debug(rpc_debug_level, "unlocked ops mutex"); - - log_debug(rpc_debug_level, "exit (*result_op=%p)", *result_op); - - return rc; -} - - -/** - * @brief Send result back to the client. - * - * This function allows the service library to send a result to - * a remote memory descriptor. The result will be packaged as - * an \ref nssi_result, then PUT on the memory descriptor of - * the remote address. - * - * @param opcode The opcode ID of the server function. - * @param return_code The return code for the server function. - * @param result Pointer to the result data structure (NULL if not used) - * @param result_addr The remote memory address of the client (where - * to PUT the result) - * - * @returns \ref NSSI_OK if successful. - */ -int nssi_send_result( - const NNTI_peer_t *caller, - const unsigned long request_id, - const int return_code, - void *result, - const NNTI_buffer_t *result_addr) -{ - int rc = NSSI_OK; - nssi_svc_op op; - - log_debug(rpc_debug_level, "enter"); - - request_args_t *args=request_args_get(caller, request_id); - - log_debug(rpc_debug_level, "args=%p", args); - - if (args->is_responseless==FALSE) { - /* lookup the service description of the opcode */ - rc = lookup_service_op(args->opcode, &op); - if (rc != NSSI_OK) { - log_warn(rpc_debug_level, "Invalid opcode=%d", args->opcode); - return rc; - } - - rc = send_result(caller, request_id, result_addr, op.encode_res, return_code, result); - if (rc != NSSI_OK) { - log_warn(rpc_debug_level, "Unable to send result to client: %s", nssi_err_str(rc)); - } - } - - return rc; -} - - -/** - * @brief Process an rpc service request. - * - * Each incoming rpc service request arrives as a chunk of xdr data. This - * method first decodes the header and arguments from the xdr data, then - * it calls the appropriate method to process the request. We define the - * parameters as void pointers because this function may execute as a thread_pool_task. - * - * - * @param args The arguments (rpc_request *). - */ - -int nssi_process_rpc_request(nssi_svc_rpc_request *rpc_req) -{ - XDR xdrs; - int rc; - nssi_svc_op svc_op; /* current operation */ - - trios_declare_timer(call_time); - - /* space for args and result (these are passed in with the header) */ - void *op_args = NULL; - void *op_res = NULL; - - nssi_request_header header; - int req_count = 0; - log_level debug_level = rpc_debug_level; - - shadow_buffer_entry *sbe=NULL; - char *shadow_data_buf =NULL; - nssi_size shadow_data_size=0; - - NNTI_buffer_t *res_addr=NULL; - - - request_args_t *req_args=NULL; - - NNTI_peer_t caller = rpc_req->caller; - char *req_buf = rpc_req->req_buf; - nssi_size short_req_len = rpc_req->short_req_len; - req_count = rpc_req->id; - - log_debug(debug_level, "req_buf=%p", req_buf); - - log_debug(debug_level, "Started processing request %d", - req_count); - - /* memory check - log memory statistics. if memory in use - * is greater than maximum memory allowed, then exit. - */ -// rc = nthread_lock(&meminfo_mutex); -// log_meminfo(rpc_debug_level); -// unsigned long main_memory_in_use = main_memory_used(); -// rc = nthread_unlock(&meminfo_mutex); -// -// log_debug(debug_level, -// "max memory check (allowed=%lukB, in_use=%lukB, %%free=%f)", -// max_mem_allowed, main_memory_in_use, -// 100.0-(((float)main_memory_in_use/(float)max_mem_allowed)*100.0)); -// if ((max_mem_allowed > 0) && -// (main_memory_in_use > max_mem_allowed)) { -// rc = NSSI_OK; -// log_error(rpc_debug_level, -// "max memory allowed exceeded (allowed=%lukB, in_use=%lukB), exiting", -// max_mem_allowed, main_memory_in_use); -// goto cleanup; -// } - - /* initialize the request header */ - memset(&header, 0, sizeof(nssi_request_header)); - - /* create an xdr memory stream from the request buffer */ - xdrmem_create( - &xdrs, - req_buf, - short_req_len, - XDR_DECODE); - - /* decode the request header */ - log_debug(debug_level, "decoding header for request %d...", - req_count); - trios_start_timer(call_time); - rc = xdr_nssi_request_header(&xdrs, &header); // this allocates an xdr_string, must use xdr_free - trios_stop_timer("xdr_nssi_request_header - decode", call_time); - if (!rc) { - log_fatal(debug_level, "failed to decode header"); - rc = NSSI_EDECODE; - goto cleanup; - } - - log_debug(debug_level, "begin processing request %d with opcode (%u)", - req_count, header.opcode); - - if (logging_debug(rpc_debug_level)) { - fprint_nssi_request_header(logger_get_file(), "header", "nssi_request_header", &header); - } - - /* See if the opcode is in our list of supported ops */ - rc = lookup_service_op(header.opcode, &svc_op); - if (rc != NSSI_OK) { - /* if we get here, there is not match */ - log_warn(debug_level, "unrecognized request: opcode=%d", - header.opcode); - rc = NSSI_EBADRPC; - goto cleanup; - } - - log_debug(LOG_OFF, "header.id=%d", header.id); - - log_info(debug_level, "Found op for opcode=%d", header.opcode); - - - op_args = calloc(1, svc_op.sizeof_args); - - op_res = calloc(1, svc_op.sizeof_res); - - /* start interval for decode args */ - trace_start_interval(trace_interval_gid, 0); - - /* initialize args and res */ - log_debug(debug_level, "NNTI_BUFFER_SIZE(&header.res_addr)==%d for request %d", - NNTI_BUFFER_SIZE(&header.res_addr), req_count); - - /* If the args fit in the header, extract them from the - * header buffer. Otherwise, get them from the client. - */ - if (!header.fetch_args) { - if (! svc_op.decode_args(&xdrs, op_args)) { - log_fatal(debug_level,"could not decode args"); - rc = NSSI_EDECODE; - goto cleanup; - } - } - else { - /* fetch the operation arguments */ - log_debug(debug_level, "fetching args for request %d", - req_count); - // is reentrant?? - trios_start_timer(call_time); - rc = fetch_args( - &caller, - &header, - svc_op.decode_args, - op_args); - trios_stop_timer("fetch_args", call_time); - if (rc != NSSI_OK) { - log_fatal(debug_level, - "unable to fetch args"); - goto cleanup; - } - } - - req_args = (request_args_t *)malloc(sizeof(request_args_t)); - req_args->opcode = header.opcode; - req_args->request_id = header.id; - req_args->arrival_time = rpc_req->arrival_time; - req_args->start_time = trios_get_time_ms(); - req_args->data_hdl=&header.data_addr; - req_args->is_responseless=header.is_responseless; - request_args_add(&caller, header.id, req_args); - - shadow_data_size=NNTI_BUFFER_SIZE(&header.data_addr); - if (header.fetch_data == TRUE) { - - req_args->shadow_data =header.data_addr; - req_args->shadow_data_hdl=&req_args->shadow_data; - - } else if ((shadow_data_size > 0) && (header.fetch_data == FALSE)) { - - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= shadow_data_size)) { - log_debug(rpc_debug_level, "using buffer queue for SHADOW buffer"); - req_args->shadow_data_hdl=trios_buffer_queue_pop(&rdma_target_bq); - assert(req_args->shadow_data_hdl); - NNTI_BUFFER_SIZE(req_args->shadow_data_hdl)=shadow_data_size; - } else { - log_debug(rpc_debug_level, "allocating buffer for SHADOW buffer"); - - req_args->shadow_data_hdl=&req_args->shadow_data; - - trios_start_timer(call_time); - rc=NNTI_alloc( - &transports[rpc_req->svc->transport_id], - shadow_data_size, - 1, - (NNTI_buf_ops_t)(NNTI_GET_SRC|NNTI_PUT_DST), - req_args->shadow_data_hdl); - trios_stop_timer("NNTI_register_memory - shadow_data_buf", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering shadow_data: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - - nssi_size data_offset=xdr_getpos(&xdrs); - - log_debug(rpc_debug_level,"extracting data (size=%d, offset=%d) from short request", shadow_data_size, data_offset); - /* copy small data into the short request */ - memcpy(NNTI_BUFFER_C_POINTER(req_args->shadow_data_hdl), req_buf+data_offset, shadow_data_size); - - sbe=(shadow_buffer_entry *)malloc(sizeof(shadow_buffer_entry)); - sbe->data_buffer =req_args->data_hdl; - sbe->shadow_buffer=req_args->shadow_data_hdl; - sbe->data_was_got =false; - sbe->data_was_put =true; - insert_shadow_buffer(sbe); - } - - if (header.is_responseless == FALSE) { - res_addr=&header.res_addr; - } - - /* end the decode args interval */ - trace_end_interval(trace_interval_gid, TRACE_RPC_DECODE, - 0, "decode request"); - - /* - ** Process the request (print warning if method fails), but - ** don't return error, because some operations are meant to fail - */ - log_debug(debug_level, "calling the server function" - " for request %d (id=%lu, opcode=%d, func=%p, obj=%p)", - req_count, header.id, header.opcode, - svc_op.func, svc_op.obj); - - // is reentrant?? - trios_start_timer(call_time); - if (svc_op.func) { - rc = svc_op.func(header.id, &caller, op_args, req_args->shadow_data_hdl, res_addr); - } else if (svc_op.obj) { - rc = svc_op.obj->doRPC(svc_op.opcode, header.id, &caller, op_args, req_args->shadow_data_hdl, res_addr); - } else { - rc = NSSI_ENOENT; - } - trios_stop_timer("svc_op", call_time); - if (rc != NSSI_OK) { - log_info(rpc_debug_level, - "user op failed: %s", - nssi_err_str(rc)); - } - - if ((shadow_data_size > 0) && (header.fetch_data == FALSE)) { - - sbe=del_shadow_buffer(req_args->shadow_data_hdl); - free(sbe); - - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= (uint32_t)shadow_data_size)) { - trios_buffer_queue_push(&rdma_target_bq, req_args->shadow_data_hdl); - } else { - trios_start_timer(call_time); - rc=NNTI_free(req_args->shadow_data_hdl); - trios_stop_timer("NNTI_unregister_memory - shadow_data_buf", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering data: %s", - nnti_err_str(rc)); - } - } - } - - /* send result back to client */ - log_debug(debug_level, "sending result for request %d " - "(%lu) back to client", req_count, header.id); - - if (rc != NSSI_OK) { - log_fatal(debug_level, "unable to send result %lu" - " for opcode=%d", header.id, header.opcode); - goto cleanup; - } - - /* free data structures created for the args and result */ - log_debug(debug_level, "xdr_freeing args for request %d", - req_count); - trios_start_timer(call_time); - xdr_free((xdrproc_t)svc_op.decode_args, (char *)op_args); - trios_stop_timer("xdr_free - args", call_time); - log_debug(debug_level, "xdr_freeing result for request %d", - req_count); - - - log_debug(debug_level, "freeing args for request %d", - req_count); - free(op_args); - log_debug(debug_level, "freeing result for request %d", - req_count); - - - /* This can be made the responsibility of the service */ - trios_start_timer(call_time); - xdr_free((xdrproc_t)svc_op.encode_res, (char *)op_res); - trios_stop_timer("xdr_free - result", call_time); - free(op_res); - - log_debug(debug_level, "result freed for request %d", - req_count); - - -cleanup: - - // release space allocated by xdr calls: leak found by valgrind - xdr_free((xdrproc_t)xdr_nssi_request_header, (char *)&header); - - log_debug(debug_level, "finished processing request %d", req_count); - - request_args_del(&caller, header.id); - - // release the data allocated for the rpc_req buffer (done in server_start) - delete [] rpc_req->req_buf; - - // delete the actual request - delete rpc_req; - - return rc; -} - - -/** - * @brief An abstract method to get data from a remote memory descriptor. - * - * The server stub uses this function to get or put data to a - * client memory descriptor. - * - * @param buf @input the buffer for the data. - * @param len @input the maximum length of the buffer. - * @param src_md @input the remote memory descriptor. - */ -int nssi_get_data( - const NNTI_peer_t *caller, - void *buf, - const int len, - const NNTI_buffer_t *data_addr) -{ - int rc = NSSI_OK; - NNTI_buffer_t rpc_msg; - NNTI_buffer_t *rpc_msg_hdl=NULL; - NNTI_work_request_t rpc_msg_wr; - NNTI_status_t status; - trios_declare_timer(call_time); - - shadow_buffer_entry *sbe=NULL; - - if (len == 0) - return rc; - - if (len < 0) - return NSSI_EINVAL; - - sbe=get_shadow_buffer(data_addr); - if (sbe != NULL) { - memcpy(buf, NNTI_BUFFER_C_POINTER(sbe->shadow_buffer), len); - sbe->data_was_got=true; - return rc; - } - - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= (uint32_t)len)) { - log_debug(rpc_debug_level, "using buffer queue for GET buffer"); - rpc_msg_hdl=trios_buffer_queue_pop(&rdma_get_bq); - assert(rpc_msg_hdl); - NNTI_BUFFER_SIZE(rpc_msg_hdl)=len; - } else { - log_debug(rpc_debug_level, "using user buffer for GET buffer"); - trios_start_timer(call_time); - rpc_msg_hdl=&rpc_msg; - rc=NNTI_register_memory( - &transports[data_addr->transport_id], - (char *)buf, - len, - 1, - NNTI_GET_DST, - rpc_msg_hdl); - trios_stop_timer("register get dest", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering data: %s", - nnti_err_str(rc)); - } - } - - trios_start_timer(call_time); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); -#endif - rc=NNTI_get( - data_addr, - 0, - len, - rpc_msg_hdl, - 0, - &rpc_msg_wr); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "nssi_get_data - NNTI_get"); -#endif - trios_stop_timer("get to get dest", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed getting data: %s", - nnti_err_str(rc)); - } - trios_start_timer(call_time); - rc=NNTI_wait( - &rpc_msg_wr, - -1, - &status); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "nssi_get_data - NNTI_wait"); -#endif - trios_stop_timer("wait for get dest", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for data: %s", - nnti_err_str(rc)); - } - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= (uint32_t)len)) { - /* copy the RDMA buffer contents into the user buffer */ - trios_start_timer(call_time); - memcpy(buf, NNTI_BUFFER_C_POINTER(rpc_msg_hdl), len); - trios_buffer_queue_push(&rdma_get_bq, rpc_msg_hdl); - trios_stop_timer("memcpy bq to get dest", call_time); - } else { - trios_start_timer(call_time); - rc=NNTI_unregister_memory(rpc_msg_hdl); - trios_stop_timer("deregister get dest", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering data: %s", - nnti_err_str(rc)); - } - } - - return rc; -} - -/** - * @brief An abstract method to put data into a remote memory descriptor. - * - * The server stub uses this function to put data to a - * client memory descriptor. - * - * @param buf @input the buffer for the data. - * @param len @input the amount of data to send. - * @param dest_md @input the remote memory descriptor. - */ -extern int nssi_put_data( - const NNTI_peer_t *caller, - const void *buf, - const int len, - const NNTI_buffer_t *data_addr, - const long timeout) -{ - int rc = NSSI_OK; - NNTI_buffer_t rpc_msg; - NNTI_buffer_t *rpc_msg_hdl=NULL; - NNTI_work_request_t rpc_msg_wr; - NNTI_status_t status; - trios_declare_timer(call_time); - - shadow_buffer_entry *sbe=NULL; - - if (len == 0) - return rc; - - if (len < 0) - return NSSI_EINVAL; - - /* if there is a shadow data buffer in use, then put the shadow data into the client's data_addr */ - sbe=get_shadow_buffer(data_addr); - if (sbe != NULL) { - data_addr = sbe->data_buffer; - } - - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= (uint32_t)len)) { - log_debug(rpc_debug_level, "using buffer queue for PUT buffer"); - rpc_msg_hdl=trios_buffer_queue_pop(&rdma_put_bq); - assert(rpc_msg_hdl); - NNTI_BUFFER_SIZE(rpc_msg_hdl)=len; - /* copy the user buffer contents into RDMA buffer */ - trios_start_timer(call_time); - memcpy(NNTI_BUFFER_C_POINTER(rpc_msg_hdl), buf, len); - trios_stop_timer("memcpy put src to bq", call_time); - } else { - log_debug(rpc_debug_level, "using user buffer for PUT buffer"); - rpc_msg_hdl=&rpc_msg; - rc=NNTI_register_memory( - &transports[data_addr->transport_id], - (char *)buf, - len, - 1, - NNTI_PUT_SRC, - rpc_msg_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering data: %s", - nnti_err_str(rc)); - } - } - trios_start_timer(call_time); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); -#endif - rc=NNTI_put( - rpc_msg_hdl, - 0, - len, - data_addr, - 0, - &rpc_msg_wr); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "nssi_put_data - NNTI_put"); -#endif - trios_stop_timer("NNTI_put - put to put dest", call_time); - if (rc != NSSI_OK) { - log_error(rpc_debug_level, "failed putting data: %s", - nnti_err_str(rc)); - } - trios_start_timer(call_time); - rc=NNTI_wait( - &rpc_msg_wr, - -1, - &status); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "nssi_put_data - NNTI_wait"); -#endif - trios_stop_timer("NNTI_wait - put to put dest", call_time); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed waiting for data: %s", - nnti_err_str(rc)); - } - if ((nssi_config.use_buffer_queue) && - (nssi_config.rdma_buffer_queue_buffer_size >= (uint32_t)len)) { - trios_buffer_queue_push(&rdma_put_bq, rpc_msg_hdl); - } else { - rc=NNTI_unregister_memory(rpc_msg_hdl); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering data: %s", - nnti_err_str(rc)); - } - } - - return rc; -} - - -/** - * @brief Register an RPC service. - * - * This method creates a named RPC service on the specified - * registry server. Along with the name, the server has to - * specify where (in the form of an \ref nssi_md) the - * client should "put" requests. - * - * @todo For now, the registry is on the same host as the service - * (registrcy_id is ignored). At some point, we need to separate - * the registry service. - * - * @param registry_id @input Process ID of the registry server. - * @param name @input Name of the service. - * @param remote_sd @input The remote service description to associate with the name. - * @param req @output The request handle (used to test for completion). - */ - /* -int nssi_register_service( - const nssi_remote_pid registry_id, - const char *name, - const nssi_service *svc, - nssi_request *req) -{ - return NSSI_ERR_NOTSUPP; -} -*/ - -/** - * @brief Add an operation to an operation list. - * - * This method adds an operation to an operation list used by - * a registered RPC service. Operations in this list must have - * the following prototype: - * - * \code - * int svc_fun(svc_args *args, svc_result *res); - * \endcode - * - * where \b svc_args is a data structure that contains all required - * arguments for the method, and \b svc_result is a structure - * that contains the result. - * - * @param svc_op @input The \ref nssi_svc_op operation description. - * @param op_list @output The operation list (modified by this function). - * - * @returns \ref NSSI_OK if the operation was successfully added. - */ -int nssi_add_svc_op( - const nssi_svc_op *svc_op, - nssi_svc_op_list **op_list) -{ - nssi_svc_op_list *new_list = NULL; - - /* create a new entry */ - new_list = (nssi_svc_op_list *) malloc(1*sizeof(nssi_svc_op_list)); - - /* initialize the entry */ - new_list->svc_op.opcode = svc_op->opcode; - new_list->svc_op.func = svc_op->func; - new_list->svc_op.sizeof_args = svc_op->sizeof_args; - new_list->svc_op.decode_args = svc_op->decode_args; - new_list->svc_op.sizeof_res = svc_op->sizeof_res; - new_list->svc_op.encode_res = svc_op->encode_res; - - /* push entry onto the front of the list */ - new_list->next = *op_list; - *op_list = new_list; - - return NSSI_OK; -} - - -/** - * @brief Initialize an RPC server. - * - * @ingroup rpc_server_api - * - * This method initializes the portals library and allocates space - * for incoming requests. - * - * @param portal_index @input the portals index to use for incoming reqs. - * @param service @output the local service descriptor (used by the server). - */ -int nssi_service_init( - const nssi_rpc_transport rpc_transport, - const int short_req_len, - nssi_service *service) -{ - int rc = NSSI_OK; - - int reqs_per_queue = 10000; - - nthread_lock_init(&supported_ops_mutex); - nthread_lock_init(&request_args_map_mutex); - nthread_lock_init(&shadow_buffer_mutex); - - /* initialize the service descriptors */ - memset(service, 0, sizeof(nssi_service)); - - service->transport_id = transports[rpc_transport].id; - /* use XDR to encode control messages */ - service->rpc_encode = NSSI_RPC_XDR; - service->svc_host = transports[rpc_transport].me; - service->req_size = short_req_len; - service->res_size = short_req_len; - - if (logging_debug(rpc_debug_level)) { - fprint_NNTI_peer(logger_get_file(), "transports[rpc_transport].me", - "nssi_service_init", &transports[rpc_transport].me); - fprint_NNTI_peer(logger_get_file(), "service->svc_host", - "nssi_service_init", &service->svc_host); - } - - - /* register trace groups (let someone else enable) */ - trace_register_group(TRACE_RPC_COUNTER_GNAME, &trace_counter_gid); - trace_register_group(TRACE_RPC_INTERVAL_GNAME, &trace_interval_gid); - - /* Register standard services */ - NSSI_REGISTER_SERVER_STUB(NSSI_OP_GET_SERVICE, rpc_get_service, void, nssi_service); - NSSI_REGISTER_SERVER_STUB(NSSI_OP_KILL_SERVICE, rpc_kill_service, nssi_kill_service_args, void); - NSSI_REGISTER_SERVER_STUB(NSSI_OP_TRACE_RESET, rpc_trace_reset, nssi_trace_reset_args, void); - - rc=NNTI_alloc( - &transports[service->transport_id], - service->req_size, - 2*reqs_per_queue, - NNTI_RECV_QUEUE, - &service->req_addr); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed registering request queue: %s", - nnti_err_str(rc)); - } - - /* copy the service description to the local service description */ - memcpy(&local_service, service, sizeof(nssi_service)); - - return rc; -} - -/** - * @brief Add operations to service. - * - * @param svc @input_type The service descriptor. - * @param ops @input_type The array operations to add to the service. - * @param len @input_type The number of operations to add. - */ -int nssi_service_add_op( - const nssi_service *unused, - const nssi_svc_op *op) -{ - int rc = NSSI_OK; - - assert(op); - - if (nthread_lock(&supported_ops_mutex)) log_warn(rpc_debug_level, "failed to get thread lock"); - - if (supported_ops.find((int)op->opcode) == supported_ops.end()) { - supported_ops[(int)op->opcode] = *op; - } - - else { - rc = NSSI_EEXIST; - } - - nthread_unlock(&supported_ops_mutex); - - return rc; -} - - - -/** - * @brief Close down an active service. - * - * @ingroup rpc_server_api - * - * Shuts down the service and releases any associated resources. - * - * @param local_sd @input The local service descriptor. - */ -int nssi_service_fini(const nssi_service *service) -{ - int rc = NSSI_OK; - - nthread_lock_fini(&supported_ops_mutex); - nthread_lock_fini(&request_args_map_mutex); - nthread_lock_fini(&shadow_buffer_mutex); - - rc=NNTI_free((NNTI_buffer_t *)&service->req_addr); - if (rc != NNTI_OK) { - log_error(rpc_debug_level, "failed unregistering request queue: %s", - nnti_err_str(rc)); - } - - time_to_die=false; - - return NSSI_OK; -} - -#define NUM_QUEUES 2 - -/** - * @brief Create a daemon process. - */ -void nssi_daemonize() -{ -#ifndef __LIBCATAMOUNT__ - int daemon_pid = fork(); - - if (daemon_pid < 0) { /* fork error */ - log_error(rpc_debug_level, "could not fork process"); - return; - } - - if (daemon_pid > 0) { /* parent exits */ - exit(0); - } - - /* child (daemon) continues */ - - /* obtain a new process groupd for the daemon */ - setsid(); - umask(0); - - /* close stdin, stdout, and stderr */ - close(0), close(1); close(2); -#endif -#if 0 - i = open("/dev/null", O_RDWR); /* open stdin */ - - dup(i); /* stdout */ - dup(i); /* stderr */ -#endif -} - -double nssi_get_request_age(const NNTI_peer_t *caller, const int req_id) -{ - double age = -1.0; - - caller_reqid cr(caller, req_id); - request_args_t *req_args = request_args_map[cr]; - - if (req_args) { - age = trios_get_time() - req_args->arrival_time; - } - else { - age = -99.99; - } - - return age; -} - - -/** - * @brief Start the RPC server using the default request processing function. - */ -int nssi_service_start(nssi_service *svc) -{ - return nssi_service_start_wfn(svc, &nssi_process_rpc_request); -} - -/** - * @brief Start the RPC server. - * - * The \b nssi_service_start implements a loop that waits for incoming - * RPC requests, then calls the process_req function pointer to - * process those requests. - * - * @param service The service descriptor. - * @param process_req A function pointer that takes an nssi_svc_rpc_request. - */ -int nssi_service_start_wfn( - nssi_service *svc, - int (*process_req)(nssi_svc_rpc_request *)) -{ - int rc = NSSI_OK; - int req_count = 0; - double t1; - double idle_time = 0; - double processing_time = 0; - trios_declare_timer(call_time); - trios_declare_timer(loop_time); - - log_level debug_level = rpc_debug_level; - - char *req_buf; - - NNTI_work_request_t req_queue_wr; - NNTI_status_t status; - - progress_callback progress_cb =NULL; - int64_t progress_timeout =2000; // needs to be reasonable (2 sec) - int64_t progress_last_time=0; - if (svc->progress_callback != 0) { - progress_cb=(progress_callback)svc->progress_callback; - progress_timeout=svc->progress_callback_timeout; - if (progress_timeout < 100) { - progress_timeout=100; - } - } - - log_debug(debug_level, "starting single-threaded rpc service"); - - /* initialize indices and counters */ - req_count = 0; /* number of reqs processed */ - - /* SIGINT (Ctrl-C) will get us out of this loop */ - while (!trios_exit_now()) { - - trios_start_timer(loop_time); - - log_debug(rpc_debug_level, "a"); - - /* exit if the time_to_die flag is set */ - if (time_to_die) { - rc = NSSI_OK; - log_info(debug_level, "responding to kill request"); - goto cleanup; - } - - /* exit if we've received our max number of reqs */ - if ((svc->max_reqs >= 0) && (req_count >= svc->max_reqs)) { - rc = NSSI_OK; - log_info(debug_level, "recved max_reqs=%d, exiting",svc->max_reqs); - goto cleanup; - } - - trace_start_interval(trace_interval_gid, 0); - - /* measure idle time */ - if (req_count > 0) { - t1 = trios_get_time(); - } - - NNTI_create_work_request( - &svc->req_addr, - &req_queue_wr); - trios_start_timer(call_time); - rc=NNTI_wait( - &req_queue_wr, - progress_timeout, - &status); - trios_stop_timer("request queue wait", call_time); - if (status.result == NNTI_ETIMEDOUT) { - - } - else if (rc != NNTI_OK) { - log_error(debug_level, "failed waiting for a request: %s", - nnti_err_str(rc)); - } - - if ((progress_cb) && - status.result == NNTI_ETIMEDOUT) { - - progress_cb(1); - progress_last_time=trios_get_time_ms(); - } - - if (trios_exit_now()) { - log_debug(debug_level, "time to exit"); - goto cleanup; - } - - if (status.result == NNTI_OK) { - req_buf = (char *)status.start+status.offset; - log_debug(debug_level, "req_buf=%p", req_buf); - - //trace_end_interval(trace_interval_gid, TRACE_THREAD_IDLE, 0, "idle time"); - log_debug(debug_level, "after end interval"); - - /* capture the idle time */ - idle_time += trios_get_time() - t1; - log_debug(debug_level, "out of job_loop"); - - /* increment the number of requests */ - req_count++; - - // this structure gets freed in the process_rpc_request function - struct rpc_request *rpc_req = new struct rpc_request(); - rpc_req->svc = svc; - rpc_req->caller = status.src; - rpc_req->id = req_count; - rpc_req->arrival_time = trios_get_time(); - - // copy the short request buffer (in case of threaded servers) - rpc_req->req_buf = new char[status.length]; // freed in process_rpc_request - memcpy(rpc_req->req_buf, req_buf, status.length); - rpc_req->short_req_len = status.length; - - - /* measure the processing time */ - t1 = trios_get_time(); - - // The process_req function is responsible for freeing the request and - // any memory associated with the request (this seems kind of risky). - - trios_start_timer(call_time); - (*process_req)(rpc_req); // call the function pointer - trios_stop_timer("process_rpc_request", call_time); - if (rc != NSSI_OK) { - /* warn only... we do not exit */ - log_warn(rpc_debug_level, "main: unable to process request."); - } - - /* measure the processing time */ - processing_time += trios_get_time() - t1; - - if ((progress_cb) && - (trios_get_time_ms() - progress_last_time) > progress_timeout) { - - progress_cb(0); - progress_last_time=trios_get_time_ms(); - } - } - NNTI_destroy_work_request( - &req_queue_wr); - - trios_stop_timer("service loop", loop_time); - } - - -cleanup: - - debug_level = rpc_debug_level; - - /* finish any tracing */ - trace_fini(); - - log_debug(debug_level, "Cleaning up..."); - - /* print out stats about the server */ - log_info(debug_level, "Exiting nssi_service_start: %d " - "reqs processed, exit_now=%d", req_count,trios_exit_now()); - - FILE *fp = logger_get_file(); - fprintf(fp, "----- SERVER STATS ---------\n"); - fprintf(fp, "\tprocessed requests = %d\n", req_count); - //fprintf(fp, "\tidle time = %g (sec)\n", idle_time); - fprintf(fp, "\tprocessing time = %g (sec)\n", processing_time); - fprintf(fp, "----------------------------\n"); - - return rc; -} diff --git a/packages/trios/libraries/support/CMakeLists.txt b/packages/trios/libraries/support/CMakeLists.txt deleted file mode 100644 index 3b8c4fd8403c..000000000000 --- a/packages/trios/libraries/support/CMakeLists.txt +++ /dev/null @@ -1,173 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -TRIBITS_SUBPACKAGE(support) - -# A) Package-specific configuration options -# - -# Initializations -SET(HEADERS "") -SET(NONINSTALLHEADERS "") -SET(SOURCES "") -SET(MODULES "") - - -#get_cmake_property(_variableNames VARIABLES) -#foreach (_variableName ${_variableNames}) -# message(STATUS "${_variableName}=${${_variableName}}") -#endforeach() - -#INCLUDE_DIRECTORIES(BEFORE SYSTEM ${${PARENT_PACKAGE}_BINARY_DIR}) - -#INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}) -#INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/..) - -message(STATUS "${${PARENT_PACKAGE_NAME}_BINARY_DIR}/Trios_config.h") - -APPEND_SET(HEADERS - ${${PARENT_PACKAGE_NAME}_BINARY_DIR}/Trios_config.h -) - -############# LOGGER ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/logger) - -APPEND_SET(HEADERS - logger/Trios_logger.h -) - -APPEND_SET(SOURCES - logger/logger.cpp -) - - -############# THREADS ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/threads) - -APPEND_SET(HEADERS - threads/Trios_threads.h - threads/Trios_threads_types.h -) - -APPEND_SET(SOURCES - threads/trios_threads.cpp -) - -############# TIMER ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/timer) - -APPEND_SET(HEADERS - timer/Trios_timer.h -) - -APPEND_SET(SOURCES - timer/timer.c - timer/Trios_Timer.cpp -) - - -############# SIGNAL ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/signal) - -APPEND_SET(HEADERS - signal/Trios_signal.h -) - -APPEND_SET(SOURCES - signal/signal.cpp -) - - -########### Trace ############### - - - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/trace) - -APPEND_SET(HEADERS - trace/Trios_trace.h -) - -APPEND_SET(NOINSTALLHEADERS - trace/SDDF.h - trace/TraceFile.h - trace/TraceGroup.h -) - -APPEND_SET(SOURCES - trace/trace.cc -) - -# See if PABLO is enabled -IF (${PACKAGE_NAME}_ENABLE_Pablo) - APPEND_SET(SOURCES trace/SDDF.cc) -ENDIF (${PACKAGE_NAME}_ENABLE_Pablo) - - -############# XDR ########################## - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/xdr) - -APPEND_SET(HEADERS - xdr/Trios_xdr.h -) - -########################### TRIOS SUPPORT LIBRARY ############### - -TRIBITS_ADD_LIBRARY( - trios_support - HEADERS ${HEADERS} - NOINSTALLHEADERS ${NOINSTALLHEADERS} - SOURCES ${SOURCES} - DEPLIBS ${DEPLIBS} - ) - - -########### install targets ############### - -# INSTALL(FILES ${MODULES} DESTINATION ${CMAKE_INSTALL_PREFIX}/include) - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/libraries/support/Readme b/packages/trios/libraries/support/Readme deleted file mode 100644 index 239409d484dd..000000000000 --- a/packages/trios/libraries/support/Readme +++ /dev/null @@ -1 +0,0 @@ -Various support diff --git a/packages/trios/libraries/support/cmake/Dependencies.cmake b/packages/trios/libraries/support/cmake/Dependencies.cmake deleted file mode 100644 index a18c0c26c97c..000000000000 --- a/packages/trios/libraries/support/cmake/Dependencies.cmake +++ /dev/null @@ -1,49 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - -SET(LIB_REQUIRED_DEP_PACKAGES) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS Pablo) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/libraries/support/logger/Readme b/packages/trios/libraries/support/logger/Readme deleted file mode 100644 index 6e1ea2a2cb90..000000000000 --- a/packages/trios/libraries/support/logger/Readme +++ /dev/null @@ -1 +0,0 @@ -A simple interface for logging events. diff --git a/packages/trios/libraries/support/logger/Trios_logger.h b/packages/trios/libraries/support/logger/Trios_logger.h deleted file mode 100644 index 7dbc03b023d9..000000000000 --- a/packages/trios/libraries/support/logger/Trios_logger.h +++ /dev/null @@ -1,223 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file trios_logger.h - * - * @brief Method prototypes for the logger API. - * - * The logger API is a simple API for logging events to a file - * (or to stderr or stdout) - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 1014 $. - * $Date: 2006-10-09 15:59:10 -0600 (Mon, 09 Oct 2006) $. - * - */ - -#ifndef _LOGGER_H_ -#define _LOGGER_H_ - -/* removes the warning when compiled with C++ */ -#ifdef __GNUC__ -#pragma GCC system_header -#endif - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef FALSE -# define FALSE (0) -#endif - -#ifndef TRUE -# define TRUE (1) -#endif - -//#define DISABLE_DEBUG_LOGGING - -#if defined(DISABLE_DEBUG_LOGGING) - -#define logging_debug(level) (0) -#define log_debug(level, ...) - -#else - -/** - * @brief Boolean function that returns TRUE if we are logging - * debug statements. - */ -#define logging_debug(level) \ - (((level == LOG_UNDEFINED) && (default_log_level >= LOG_DEBUG)) \ - || ((level != LOG_UNDEFINED) && (level >= LOG_DEBUG))) - -/** - * @brief Inline function that outputs a DEBUG - * message to the log file. - * - * @param level The log level to use. - * @param args A formatted message (like printf). - */ -#define log_debug(level, ...) if (logging_debug(level)) \ - log_output("DEBUG",__FUNCTION__,__FILE__,__LINE__, ## __VA_ARGS__) - -#endif - -/** - * @brief Boolean function that returns TRUE if we are logging - * info statements. - */ -#define logging_info(level) \ - (((level == LOG_UNDEFINED) && (default_log_level >= LOG_INFO)) \ - || ((level != LOG_UNDEFINED) && (level >= LOG_INFO))) - -/** - * @brief Inline function that outputs an INFO - * message to the log file. - * - * @param level The log level to use. - * @param args A formatted message (like printf). - */ -#define log_info(level, ...) if (logging_info(level)) \ - log_output("INFO",__FUNCTION__,__FILE__,__LINE__, ## __VA_ARGS__) - -/** - * @brief Boolean function that returns TRUE if we are logging - * warning statements. - */ -#define logging_warn(level) \ - (((level == LOG_UNDEFINED) && (default_log_level >= LOG_WARN)) \ - || ((level != LOG_UNDEFINED) && (level >= LOG_WARN))) - -/** - * @brief Inline function that outputs a WARN - * message to the log file. - * - * @param level The log level to use. - * @param args A formatted message (like printf). - */ -#define log_warn(level, ...) if (logging_warn(level)) \ - log_output("WARN",__FUNCTION__,__FILE__,__LINE__, ## __VA_ARGS__) - -/** - * @brief Boolean function that returns TRUE if we are logging - * error statements. - */ -#define logging_error(level) \ - (((level == LOG_UNDEFINED) && (default_log_level >= LOG_ERROR)) \ - || ((level != LOG_UNDEFINED) && (level >= LOG_ERROR))) - -/** - * @brief Inline function that outputs an ERROR - * message to the log file. - * - * @param level The log level to use. - * @param args A formatted message (like printf). - */ -#define log_error(level, ...) if (logging_error(level)) \ - log_output("ERROR",__FUNCTION__,__FILE__,__LINE__, ## __VA_ARGS__) - -/** - * @brief Boolean function that returns TRUE if we are logging - * error statements. - */ -#define logging_fatal(level) \ - (((level == LOG_UNDEFINED) && (default_log_level >= LOG_FATAL)) \ - || ((level != LOG_UNDEFINED) && (level >= LOG_FATAL))) - - -/** - * @brief Inline function that outputs a FATAL - * message to the log file. - * - * @param level The log level to use. - * @param args A formatted message (like printf). - */ -#define log_fatal(level, ...) if (logging_fatal(level)) \ - log_output("FATAL",__FUNCTION__,__FILE__,__LINE__, ## __VA_ARGS__ ) - -enum log_level { - LOG_UNDEFINED = -1, - LOG_OFF = 0, - LOG_FATAL = 1, - LOG_ERROR = 2, - LOG_WARN = 3, - LOG_INFO = 4, - LOG_DEBUG = 5, - LOG_ALL = 6 -}; -typedef enum log_level log_level; - -extern log_level default_log_level; - -/* the functions */ - -#if defined(__STDC__) || defined(__cplusplus) - -extern void logger_mutex_lock(void); -extern void logger_mutex_unlock(void); -extern int logger_init(const log_level debug_level, const char *file); -extern int logger_not_initialized(void); -extern void logger_set_file(FILE *); -extern FILE *logger_get_file(void); -extern void logger_set_default_level(const log_level); -extern log_level logger_get_default_level(void); - -void log_output(const char *prefix, - const char *func_name, - const char *file_name, - const int line_no, - const char *msg, ...); - -extern int logger_fini(void); - -#endif - - -#ifdef __cplusplus -} -#endif - -#endif /* !LOGGER_H_ */ diff --git a/packages/trios/libraries/support/logger/logger.cpp b/packages/trios/libraries/support/logger/logger.cpp deleted file mode 100644 index 52d32d885245..000000000000 --- a/packages/trios/libraries/support/logger/logger.cpp +++ /dev/null @@ -1,277 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file logger.c - * - * @brief This file contains method defintions for the logger API. - * - * The logger API is a simple API for logging events to a file - * (or to stderr or stdio) - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 1576 $. - * $Date: 2007-10-01 17:31:49 -0600 (Mon, 01 Oct 2007) $. - * - */ - -#include "Trios_config.h" - -#include -#include -#include -#include -#include -#include - -#ifdef HAVE_TRIOS_UNISTD_H -#include -#endif -#ifdef HAVE_TRIOS_SYSCALL_H -#include -#endif - -#include "Trios_logger.h" -#include "Trios_threads.h" - -static bool mutex_initialized = false; -static nthread_lock_t logger_mutex; - -void logger_mutex_lock() -{ - if (!mutex_initialized) { - if (nthread_lock_init(&logger_mutex) == -1) { - fprintf(stderr, "nthread_lock_init failed.\n"); - fflush(stderr); - return; - } - - mutex_initialized = true; - } - - nthread_lock(&logger_mutex); -} -void logger_mutex_unlock() -{ - if (!mutex_initialized) { - fprintf(stderr, "logger_mutex_unlock: mutex not intialized.\n"); - fflush(stderr); - return; - } - - nthread_unlock(&logger_mutex); -} - -log_level default_log_level = LOG_WARN; -static FILE *log_file = NULL; - - - -/** - * @brief Initialize the logging capabilities. - * - * @param level Log level (LOG_ALL, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL, or LOG_OFF) - * @param logfile File name of the logging output. - * - */ -int logger_init(const log_level debug_level, const char *logfile) -{ - int rc = 0; - - if (!mutex_initialized) { - if (nthread_lock_init(&logger_mutex) == -1) { - fprintf(stderr, "nthread_lock_init failed.\n"); - fflush(stderr); - return(-1); - } - - mutex_initialized = true; - } - - /* initialize the default debug level */ - if (debug_level == 0) - logger_set_default_level(LOG_OFF); - else if (debug_level > 5) - logger_set_default_level(LOG_ALL); - else { - int new_level = (int)debug_level - LOG_OFF; - logger_set_default_level((log_level)new_level); - } - - /* initialize the logfile */ - if ((logfile == NULL) || (logfile[0] == '\0')) { - logger_set_file(stdout); - } - else if (strcasecmp("stdout", logfile) == 0) { - logger_set_file(stdout); - } - - else if (strcasecmp("stderr", logfile) == 0) { - logger_set_file(stderr); - } - - else { - FILE *fp = fopen(logfile, "w+"); - if (fp == NULL) { - fprintf(stderr, "could not create log file \"%s\"\n",logfile); - return(-1); - } - else { - logger_set_file(fp); - } - } - - return(rc); -} - -int logger_not_initialized() -{ - return log_file == NULL; -} - -/** - * @brief Set the file for the log information. - */ -void logger_set_file(FILE *newfile) -{ - log_file = newfile; -} - -FILE *logger_get_file() -{ - if (!log_file) - return stdout; - else - return log_file; -} - - -/** - * @brief Set the default log level. - * - * The different log levels are LOG_ALL, LOG_DEBUG, LOG_INFO, LOG_WARN, - * LOG_ERROR, LOG_FATAL, and LOG_OFF. - */ -void logger_set_default_level(const log_level newlevel) -{ - default_log_level = newlevel; -} - -/** - * @brief Return the default log level. - */ -log_level logger_get_default_level(void) -{ - return default_log_level; -} - -/** - * @brief Output a log message. - * - * This method should be called by one of the inline - * methods log_debug, log_info, log_warn, log_error, or - * log_fatal. - */ -void log_output(const char *prefix, - const char *func_name, - const char *file_name, - const int line_num, - const char *msg, - ...) -{ - va_list ap; - const char *file; - char buf1[256]; - char buf2[256]; - - if (logger_not_initialized()) { - logger_init(LOG_ERROR, NULL); - } - - /* path from last '/' */ - file = strrchr(file_name, '/'); - - va_start(ap, msg); - -#ifdef HAVE_TRIOS_GETTID - sprintf(buf1, "%s [%s:%s:%d:t%lu]: ", - prefix, - func_name, - (file == NULL) ? file_name : &(file[1]), - line_num, - syscall(SYS_gettid)); -#else - sprintf(buf1, "%s [%s:%s:%d]: ", - prefix, - func_name, - (file == NULL) ? file_name : &(file[1]), - line_num); -#endif - - vsprintf(buf2, msg, ap); - logger_mutex_lock(); - fprintf(log_file, "%s %s\n", buf1, buf2); - logger_mutex_unlock(); - va_end(ap); - fflush(log_file); -} - -/** - * @brief Finalize the logging capabilities. - * - */ -int logger_fini(void) -{ - int rc = 0; - - if ((log_file) && (log_file != stdout) && (log_file != stderr)) { - fclose(log_file); - } - log_file=NULL; - - if (mutex_initialized) { - nthread_lock_fini(&logger_mutex); - mutex_initialized = false; - } - - return rc; -} diff --git a/packages/trios/libraries/support/signal/Trios_signal.h b/packages/trios/libraries/support/signal/Trios_signal.h deleted file mode 100644 index 8d399d83e095..000000000000 --- a/packages/trios/libraries/support/signal/Trios_signal.h +++ /dev/null @@ -1,100 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file Trios_signal.h - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 756 $ - * $Date: 2006-06-27 14:12:27 -0600 (Tue, 27 Jun 2006) $ - * - */ - -#ifndef _RPC_SIGNAL_H_ -#define _RPC_SIGNAL_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - /** - * @brief Install signal handlers. - * - * Along with the signal handlers, we pass optional - * mutex/cond variable pair so we can signal any - * threads waiting for the abort call. - */ - extern int trios_install_sighandler(); - - - /** - * @brief Return the exit_now variable. If set, - * it is time to exit the service. - */ - extern int trios_exit_now(); - - - /** - * @brief Cleanly abort the running service. - * - * The abort function kills a running service by sending a - * SIGINT signal to the running process. If the service - * has already started, the signal is caught by the - * signal handler. - */ - extern void trios_abort(); - - -#else /* K&R C */ - -#endif /* K&R C */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/support/signal/signal.cpp b/packages/trios/libraries/support/signal/signal.cpp deleted file mode 100644 index b1e1c162421b..000000000000 --- a/packages/trios/libraries/support/signal/signal.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file trios_signal.c - * - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 560 $ - * $Date: 2006-02-28 14:02:02 -0700 (Tue, 28 Feb 2006) $ - * - */ - -#include "Trios_config.h" - -#include -#include - -#include "Trios_logger.h" -#include "Trios_signal.h" -#include "Trios_threads.h" - -#include -#include - -using namespace std; - - -/* local variables */ -static int volatile _exit_now = 0; - -/* --------------------- Private methods ------------------- */ - -static void sighandler(int sig) -{ - log_warn(LOG_UNDEFINED, "Caught signal %d, setting exit_now flag", sig); - trios_abort(); - //exit(sig); -} - - -/** - * @brief Install signal handlers. - */ -int trios_install_sighandler() -{ - struct sigaction new_action, old_action; - - new_action.sa_handler = sighandler; - sigemptyset (&new_action.sa_mask); - new_action.sa_flags = 0; - - sigaction (SIGINT, NULL, &old_action); - if (old_action.sa_handler != SIG_IGN) { - sigaction (SIGINT, &new_action, NULL); - } - sigaction (SIGHUP, NULL, &old_action); - if (old_action.sa_handler != SIG_IGN) { - sigaction (SIGHUP, &new_action, NULL); - } - sigaction (SIGTERM, NULL, &old_action); - if (old_action.sa_handler != SIG_IGN) { - sigaction (SIGTERM, &new_action, NULL); - } - sigaction (SIGABRT, NULL, &old_action); - if (old_action.sa_handler != SIG_IGN) { - sigaction (SIGABRT, &new_action, NULL); - } - return 0; -} - - -/** - * @brief Return the exit_now variable. If set, - * it is time to exit the service. - */ -int trios_exit_now() { - return _exit_now; -} - - -/** - * @brief Cleanly abort the running service. - * - * The abort function kills a running service by sending a - * SIGINT signal to the running process. If the service - * has already started, the signal is caught by the - * signal handler. - */ -void trios_abort() -{ - log_level debug_level = LOG_UNDEFINED; - - /* kill(0,SIGINT) */ - log_debug(debug_level, "Received abort()... setting exit_now flag"); - _exit_now = 1; -} diff --git a/packages/trios/libraries/support/threads/Trios_threads.h b/packages/trios/libraries/support/threads/Trios_threads.h deleted file mode 100644 index 19588a792828..000000000000 --- a/packages/trios/libraries/support/threads/Trios_threads.h +++ /dev/null @@ -1,109 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * TRIOS does not have any internal threading, but it should run in a - * multithreaded environment. The nthread library provides a single API - * for locks and atomic counters. - */ - -#ifndef _TRIOS_THREADS_H_ -#define _TRIOS_THREADS_H_ - -#include "Trios_config.h" - -#include "Trios_logger.h" -#include "Trios_threads_types.h" - - -#ifdef __cplusplus -extern "C" { -#endif - -extern log_level thread_debug_level; - - - -int nthread_lock_init( - nthread_lock_t *lock); -int nthread_lock( - nthread_lock_t *lock); -int nthread_unlock( - nthread_lock_t *lock); -int nthread_lock_fini( - nthread_lock_t *lock); - -int nthread_cond_init( - nthread_cond_t *condvar); -int nthread_wait( - nthread_cond_t *condvar, - nthread_lock_t *lock); -int nthread_timedwait( - nthread_cond_t *condvar, - nthread_lock_t *lock, - uint64_t timeout); -int nthread_signal( - nthread_cond_t *condvar); -int nthread_broadcast( - nthread_cond_t *condvar); -int nthread_cond_fini( - nthread_cond_t *condvar); - -int nthread_counter_init( - nthread_counter_t *c); -int64_t nthread_counter_increment( - nthread_counter_t *c); -int64_t nthread_counter_decrement( - nthread_counter_t *c); -int64_t nthread_counter_read( - nthread_counter_t *c); -int64_t nthread_counter_set( - nthread_counter_t *c, - int64_t new_value); -int nthread_counter_fini( - nthread_counter_t *c); - -#ifdef __cplusplus -} -#endif - -#endif /* _TRIOS_THREADS_H_ */ diff --git a/packages/trios/libraries/support/threads/Trios_threads_types.h b/packages/trios/libraries/support/threads/Trios_threads_types.h deleted file mode 100644 index b4f256586e76..000000000000 --- a/packages/trios/libraries/support/threads/Trios_threads_types.h +++ /dev/null @@ -1,90 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * TRIOS does not have any internal threading, but it should run in a - * multithreaded environment. The nthread library provides a single API - * for locks and atomic counters. - */ - -#ifndef _TRIOS_THREADS_TYPES_H_ -#define _TRIOS_THREADS_TYPES_H_ - -#include "Trios_config.h" - -#include - -#if defined(HAVE_TRIOS_PTHREAD_H) -#include -#endif - -#if defined(HAVE_TRIOS_SEMAPHORE_H) -#include -#endif - -typedef struct { - char *name; -#if defined(HAVE_TRIOS_PTHREAD_MUTEX_T) - pthread_mutex_t lock; -#elif defined(HAVE_TRIOS_SEM_T) - sem_t lock; - sem_t *lock_ptr; -#else -#warning No locking mechanism available on this system. -#endif -} nthread_lock_t; - -typedef struct { - nthread_lock_t lock; - int64_t value; -} nthread_counter_t; - -typedef struct { -#if defined(HAVE_TRIOS_PTHREAD_COND_T) - pthread_cond_t condvar; -#elif defined(HAVE_TRIOS_SEM_T) -/* #warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. */ -#endif -} nthread_cond_t; - - -#endif /* _TRIOS_THREADS_TYPES_H_ */ diff --git a/packages/trios/libraries/support/threads/trios_threads.cpp b/packages/trios/libraries/support/threads/trios_threads.cpp deleted file mode 100644 index 72527c399b6d..000000000000 --- a/packages/trios/libraries/support/threads/trios_threads.cpp +++ /dev/null @@ -1,530 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * TRIOS does not have any internal threading, but it should run in a - * multithreaded environment. The nthread library provides a single API - * for locks and atomic counters. - */ - -#include "Trios_config.h" - -#include -#include - -#ifdef HAVE_TRIOS_MALLOC_H -#include -#endif - -#include -#include -#include - -#include -#include - -#if defined(HAVE_TRIOS_PTHREAD_H) -#include -#endif - -#if defined(HAVE_TRIOS_SEMAPHORE_H) -#include -#endif - -#include "Trios_logger.h" -#include "Trios_threads.h" - -#define _DEBUG_LOCKS_ -#undef _DEBUG_LOCKS_ - -/* set to LOG_UNDEFINED -- log commands will use default level */ -log_level thread_debug_level = LOG_UNDEFINED; - - -int nthread_lock_init( - nthread_lock_t *lock) -{ - int rc=0; - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_lock_init: initializing lock(%p)\n", (void*)lock); - fflush(logger_get_file()); -#endif - -#if defined(HAVE_TRIOS_PTHREAD_MUTEX_INIT) - pthread_mutex_init(&lock->lock, NULL); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) - rc=sem_init(&lock->lock, 0, 1); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_lock_init: sem_init failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - lock->lock_ptr=NULL; - return(-1); - } - lock->lock_ptr=&lock->lock; -#elif defined(HAVE_TRIOS_NAMED_SEMAPHORES) - bool done=false; - do { - lock->name=tempnam("/tmp", "trios."); - lock->lock_ptr=sem_open(lock->name+4, O_CREAT|O_EXCL, 0600, 1); - if ((lock->lock_ptr == SEM_FAILED) && (errno == EEXIST)) { - done=false; - } else { - done=true; - } - } while (!done); - - if (lock->lock_ptr == SEM_FAILED) { - fprintf(logger_get_file(), "nthread_lock_init: sem_open failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - lock->lock_ptr=NULL; - return(-1); - } -#else -#warning No locking mechanism available on this system. -#endif - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_lock_init: initialized lock(%p), lock->name(%s)\n", (void*)lock, lock->name); - fflush(logger_get_file()); -#endif - - return(rc); -} - -int nthread_lock( - nthread_lock_t *lock) -{ - int rc=0; - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_lock: locking lock(%p)\n", (void*)lock); - fflush(logger_get_file()); -#endif - -#if defined(HAVE_TRIOS_PTHREAD_MUTEX_LOCK) - pthread_mutex_lock(&lock->lock); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) - if (lock->lock_ptr == NULL) { - fprintf(logger_get_file(), "nthread_lock: lock not initialized\n"); - fflush(logger_get_file()); - return(-1); - } - - rc=sem_wait(lock->lock_ptr); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_lock: sem_wait failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } -#else -#warning No locking mechanism available on this system. -#endif - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_lock: locked lock(%p)\n", (void*)lock); - fflush(logger_get_file()); -#endif - - return(rc); -} - -int nthread_unlock( - nthread_lock_t *lock) -{ - int rc=0; - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_unlock: unlocking lock(%p)\n", (void*)lock); - fflush(logger_get_file()); -#endif - -#if defined(HAVE_TRIOS_PTHREAD_MUTEX_UNLOCK) - pthread_mutex_unlock(&lock->lock); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) - if (lock->lock_ptr == NULL) { - fprintf(logger_get_file(), "nthread_unlock: lock not initialized\n"); - fflush(logger_get_file()); - return(-1); - } - - rc=sem_post(lock->lock_ptr); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_unlock: sem_post failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } -#else -#warning No locking mechanism available on this system. -#endif - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_unlock: unlocked lock(%p)\n", (void*)lock); - fflush(logger_get_file()); -#endif - - return(rc); -} - - -int nthread_lock_fini( - nthread_lock_t *lock) -{ - int rc=0; - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_lock_fini: finalizing lock(%p), lock->name(%s)", (void*)lock, lock->name); - fflush(logger_get_file()); -#endif - -#if defined(HAVE_TRIOS_PTHREAD_MUTEX_DESTROY) - // do nothing -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) - if (lock->lock_ptr == NULL) { - fprintf(logger_get_file(), "nthread_lock_fini: lock not initialized\n"); - fflush(logger_get_file()); - return(-1); - } -#endif - -#if defined(HAVE_TRIOS_PTHREAD_MUTEX_DESTROY) - pthread_mutex_destroy(&lock->lock); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) - rc=sem_destroy(lock->lock_ptr); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_lock_fini: sem_destroy failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } -#elif defined(HAVE_TRIOS_NAMED_SEMAPHORES) - rc=sem_close(lock->lock_ptr); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_lock_fini: sem_close failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } - rc=sem_unlink(lock->name+4); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_lock_fini: sem_unlink failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } - - free(lock->name); - lock->name=NULL; -#else -#warning No locking mechanism available on this system. -#endif - -#ifdef _DEBUG_LOCKS_ - fprintf(logger_get_file(), "nthread_lock_fini: finalized lock(%p), lock->name(%s)", (void*)lock, lock->name); - fflush(logger_get_file()); -#endif - - return(rc); -} - - -int nthread_cond_init( - nthread_cond_t *condvar) -{ - int rc=0; - -#if defined(HAVE_TRIOS_PTHREAD_COND_INIT) - pthread_cond_init(&condvar->condvar, NULL); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) -//#warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. -#endif - - return(rc); -} - -int nthread_wait( - nthread_cond_t *condvar, - nthread_lock_t *lock) -{ - int rc=0; - -#if defined(HAVE_TRIOS_PTHREAD_COND_WAIT) - pthread_cond_wait(&condvar->condvar, &lock->lock); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) -//#warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. -#endif - - return(rc); -} - -int nthread_timedwait( - nthread_cond_t *condvar, - nthread_lock_t *lock, - uint64_t timeout) -{ - int rc=0; - -#if defined(HAVE_TRIOS_PTHREAD_COND_TIMEDWAIT) - struct timespec abstime; - - int sec=timeout/1000; - long ms =timeout-sec*1000; - - clock_gettime(CLOCK_REALTIME, &abstime); - - // perform the addition - abstime.tv_nsec+=ms*1000000; - // adjust the time - abstime.tv_sec+=abstime.tv_nsec/1000000000 + sec; - abstime.tv_nsec=abstime.tv_nsec%1000000000; - - rc=pthread_cond_timedwait(&condvar->condvar, &lock->lock, &abstime); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) -//#warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. -#endif - - return(rc); -} - -int nthread_signal( - nthread_cond_t *condvar) -{ - int rc=0; - -#if defined(HAVE_TRIOS_PTHREAD_COND_SIGNAL) - pthread_cond_signal(&condvar->condvar); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) -//#warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. -#endif - - return(rc); -} - -int nthread_broadcast( - nthread_cond_t *condvar) -{ - int rc=0; - -#if defined(HAVE_TRIOS_PTHREAD_COND_BROADCAST) - pthread_cond_broadcast(&condvar->condvar); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) -//#warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. -#endif - - return(rc); -} - -int nthread_cond_fini( - nthread_cond_t *condvar) -{ - int rc=0; - -#if defined(HAVE_TRIOS_PTHREAD_COND_DESTROY) - pthread_cond_destroy(&condvar->condvar); -#elif defined(HAVE_TRIOS_UNNAMED_SEMAPHORES) || defined(HAVE_TRIOS_NAMED_SEMAPHORES) -//#warning The semaphores implementation of NSSI threads does not have conditional variables. Sorry. -#endif - - return(rc); -} - - -int nthread_counter_init( - nthread_counter_t *c) -{ - int rc=0; - -// log_debug(thread_debug_level, "nthread_counter_init(STUB)"); - rc=nthread_lock_init(&c->lock); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_counter_init: nthread_lock_init failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } - - if (nthread_lock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_init: failed to lock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - goto cleanup; - } - - c->value = 0; - - if (nthread_unlock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_init: failed to unlock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - } - -cleanup: - return rc; -} - -int64_t nthread_counter_increment( - nthread_counter_t *c) -{ - int64_t t=0; - -// log_debug(thread_debug_level, "nthread_counter_increment(STUB)"); - - if (nthread_lock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_increment: failed to lock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - t = -1; - goto cleanup; - } - - t = c->value; - c->value++; - - if (nthread_unlock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_increment: failed to unlock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - } - -cleanup: - return t; -} - -int64_t nthread_counter_decrement( - nthread_counter_t *c) -{ - int64_t t=0; - -// log_debug(thread_debug_level, "nthread_counter_decrement(STUB)"); - - if (nthread_lock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_decrement: failed to lock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - t = -1; - goto cleanup; - } - - t = c->value; - c->value--; - - if (nthread_unlock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_decrement: failed to unlock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - } - -cleanup: - return t; -} - -int64_t nthread_counter_read( - nthread_counter_t *c) -{ - int64_t t=0; - -// log_debug(thread_debug_level, "nthread_counter_read(STUB)"); - - if (nthread_lock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_read: failed to lock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - t = -1; - goto cleanup; - } - - t = c->value; - - if (nthread_unlock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_read: failed to unlock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - } - -cleanup: - return t; -} - -int64_t nthread_counter_set( - nthread_counter_t *c, - int64_t new_value) -{ - int64_t t=0; - -// log_debug(thread_debug_level, "nthread_counter_set(STUB)"); - - if (nthread_lock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_set: failed to lock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - t = -1; - goto cleanup; - } - - t = c->value; - c->value=new_value; - - if (nthread_unlock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_set: failed to unlock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - } - -cleanup: - return t; -} - -int nthread_counter_fini( - nthread_counter_t *c) -{ - int rc=0; - -// log_debug(thread_debug_level, "nthread_counter_fini(STUB)"); - if (nthread_lock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_fini: failed to lock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - goto cleanup; - } - - c->value = 0; - - if (nthread_unlock(&c->lock) != 0) { - fprintf(logger_get_file(), "nthread_counter_fini: failed to unlock the counter lock: %s\n", strerror(errno)); - fflush(logger_get_file()); - } - - rc=nthread_lock_fini(&c->lock); - if (rc == -1) { - fprintf(logger_get_file(), "nthread_counter_fini: nthread_lock_fini failed: %s\n", strerror(errno)); - fflush(logger_get_file()); - return(-1); - } - -cleanup: - return rc; -} diff --git a/packages/trios/libraries/support/timer/Trios_Timer.cpp b/packages/trios/libraries/support/timer/Trios_Timer.cpp deleted file mode 100644 index 7c7be7821321..000000000000 --- a/packages/trios/libraries/support/timer/Trios_Timer.cpp +++ /dev/null @@ -1,132 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include "Trios_timer.h" -#include -#include -#include -#include -#include - - -double Trios::GetTime() -{ - return trios_get_time(); -} - -long Trios::GetTimeMS() -{ - return trios_get_time_ms(); -} - -long Trios::GetTimeNS() -{ - return trios_get_time_ns(); -} - - -long Trios::GetTimeUS() -{ - return trios_get_time_us(); -} - - -int Trios::WriteTimings( - const std::string &fname, - const std::string &header, - const std::vector &timings_desc, - const std::vector &timings) -{ - if (fname.empty()) - return WriteTimings(std::cout, header, timings_desc, timings, true); - - // If the file does not exist, put the complete header - bool write_header = true; - std::fstream fin; - fin.open(fname.c_str(), std::ios::in); - if (fin.is_open()) { - write_header = false; - } - fin.close(); - - // Open the file for writing - std::ofstream fout; - fout.open(fname.c_str(), std::ios::out | std::ios::app); - WriteTimings(fout, header, timings_desc, timings, write_header); - fout.close(); - return 0; -} - -int Trios::WriteTimings(std::ostream &out, - const std::string &header, - const std::vector &timing_desc, - const std::vector &timings, - const bool write_header) -{ - using namespace std; - - string prefix("%"); - string separator(",\t"); - - if (write_header) { - time_t rawtime = time(NULL); - out << prefix << setfill('-') << setw(52) << " " << std::endl; - out << prefix << " Timings: " << ctime(&rawtime); - if (!header.empty()) - out << prefix << header << endl; - - out << prefix << " Column IDS:" << endl; - for (int i=0; i<(int)timing_desc.size(); i++) { - out << prefix << setfill(' ') << setw(6) << (i+1) << ": " << timing_desc[i] << endl; - } - out << prefix << setfill('-') << setw(52) << " " << endl; - } - - // just write timing information - for (int i=0; i<(int)timings.size()-1; i++) { - out.precision(5); - out << scientific << timings[i] << separator; - } - out << timings[timings.size()-1] << endl; - return 0; -} diff --git a/packages/trios/libraries/support/timer/Trios_timer.h b/packages/trios/libraries/support/timer/Trios_timer.h deleted file mode 100644 index 6c01551a3f37..000000000000 --- a/packages/trios/libraries/support/timer/Trios_timer.h +++ /dev/null @@ -1,137 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** @file time.h - * - * @brief API for calculating statistics. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 406 $. - * $Date: 2005-10-07 15:08:29 -0600 (Fri, 07 Oct 2005) $. - * - */ - -#ifndef _TRIOS_TIMER_H_ -#define _TRIOS_TIMER_H_ - -#include "Trios_logger.h" - -#include - -#ifdef __cplusplus - -#include -#include -#include - - -namespace Trios { - -double GetTime(); -long GetTimeMS(); -long GetTimeNS(); -long GetTimeUS(); - -int WriteTimings(const std::string &fname, - const std::string &header, - const std::vector &timing_desc, - const std::vector &timings); - -int WriteTimings(std::ostream &out, - const std::string &header, - const std::vector &timing_desc, - const std::vector &timings, - const bool write_header=true); - -} -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - -extern log_level timer_debug_level; - - extern double trios_get_time(); - extern long trios_get_time_ns(); - extern long trios_get_time_ms(); - extern long trios_get_time_us(); - extern int trios_timer_test(); - extern const char *trios_timer_getimpl(); - - -#if defined(TRIOS_USE_TIMERS) - -/* always use this macro to declare timer variables */ -#define trios_declare_timer(timer_var) double timer_var; - -#define trios_start_timer(timer) { timer = trios_get_time(); } -#define trios_stop_timer(name, timer) { timer = trios_get_time() - timer; log_info(timer_debug_level, "%s Time = %10.8f", name, timer); } - -#define trios_start_delay_timer(timer) { timer = trios_get_time(); } -#define trios_stop_delay_timer(timer) { timer = trios_get_time() - timer; } -#define trios_log_delay_timer(name, timer) { log_info(timer_debug_level, "%s Time = %10.8f", name, timer); } - -#else - -#define trios_declare_timer(t) - -#define trios_start_timer(timer) {} -#define trios_stop_timer(name, timer) {} - -#define trios_start_delay_timer(timer) {} -#define trios_stop_delay_timer(timer) {} -#define trios_log_delay_timer(name, timer) {} - -#endif - -#endif - - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/libraries/support/timer/timer.c b/packages/trios/libraries/support/timer/timer.c deleted file mode 100644 index 6817d6b9527f..000000000000 --- a/packages/trios/libraries/support/timer/timer.c +++ /dev/null @@ -1,370 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file timer.c - * - * @brief A portable timer for Trios. - * - * @author Ron Oldfield (raoldfi@sandia.gov) - * @version $Revision: 406 $ - * @date $Date: 2005-10-07 15:08:29 -0600 (Fri, 07 Oct 2005) $ - * - */ - -#include "Trios_config.h" - -#include /* for fabsl() */ -#include /* for llabs() */ -#include -#include -#include -#include "Trios_logger.h" - -log_level timer_debug_level = LOG_UNDEFINED; - -/*-------------------- Global Variables ------------------ */ - -/* Even though configure scripts find clock_gettime, it is - * not supported by Catamount and will cause problems if used. - */ -#ifdef __LIBCATAMOUNT__ -#undef HAVE_TRIOS_CLOCK_GETTIME -#endif - -#undef USING_PAPI -#undef USING_MACH_ABSOLUTE_TIME -#undef USING_CLOCK_GETTIME -#undef USING_GETTIMEOFDAY - - -#if defined(HAVE_TRIOS_PAPI) -#define USING_PAPI -#include -long_long init_usec, init_cycles; - - -#elif defined(HAVE_TRIOS_MACH_ABSOLUTE_TIME) -#define USING_MACH_ABSOLUTE_TIME -#include - -#elif defined(HAVE_TRIOS_CLOCK_GETTIME) -#define USING_CLOCK_GETTIME -#include /* values for various timers */ - -/* Options for clockid: - * - CLOCK_REALTIME, - * - CLOCK_MONOTONIC, - * - CLOCK_PROCESS_CPUTIME_ID, - * - CLOCK_THREAD_CPUTIME_ID - */ -static const clockid_t clockid = CLOCK_REALTIME; -static struct timespec ts_init; -static struct timespec ts_res; - -#elif defined(HAVE_TRIOS_GETTIMEOFDAY) -#define USING_GETTIMEOFDAY -#include -static struct timeval tv_init; - - -#else -#error No supported timers. -#endif - - -/*-------------------- Local Functions ------------------ */ - -static void _timer_premain (void) __attribute__ ((constructor)); -static void init_timer(); - -static void _timer_premain(void) -{ - init_timer(); -} - -static void init_timer() { - static int initialized = 0; - if (!initialized) { - -#ifdef USING_PAPI - PAPI_library_init(PAPI_VER_CURRENT); - - init_usec = PAPI_get_real_usec(); - init_cycles = PAPI_get_real_cyc(); -#endif - -#ifdef USING_CLOCK_GETTIME - clock_getres(clockid, &ts_res); - clock_gettime(clockid, &ts_init); -#endif - -#ifdef USING_GETTIMEOFDAY - gettimeofday(&tv_init, NULL); -#endif - - initialized = 1; - } -} - - -#if defined(USING_CLOCK_GETTIME) -static uint64_t ts_diff_nsec(struct timespec start, struct timespec end) -{ - struct timespec diff; - uint64_t result = 0; - - diff.tv_sec = end.tv_sec - start.tv_sec; - diff.tv_nsec = end.tv_nsec - start.tv_nsec; - - result = diff.tv_sec*1e9 + diff.tv_nsec; - return result; -} -#endif - -#ifdef USING_GETTIMEOFDAY - -static uint64_t tv_diff_us(struct timeval start, struct timeval end) -{ - struct timeval diff; - uint64_t result = 0; - - diff.tv_sec = end.tv_sec - start.tv_sec; - diff.tv_usec = end.tv_usec - start.tv_usec; - - result = diff.tv_sec*1000000 + diff.tv_usec; - return result; -} -#endif - - -/*-------------------- Timer API ------------------ */ - -/** - * @brief Return the time in nanoseconds from first call to trios_get_time_*. - */ -uint64_t trios_get_time_ns() -{ - uint64_t result = 0; - -#ifdef USING_PAPI - - long_long cur_usec, diff_usec; - - init_timer(); - cur_usec = PAPI_get_real_usec(); - diff_usec = cur_usec - init_usec; - - /* to convert to ns, multiply by 1000 */ - result = diff_usec * 1000; -#endif - -#ifdef USING_CLOCK_GETTIME - struct timespec tp; - - init_timer(); - clock_gettime(clockid, &tp); - result = ts_diff_nsec(ts_init, tp); -#endif - -#ifdef USING_MACH_ABSOLUTE_TIME - static mach_timebase_info_data_t info = {0,0}; - - init_timer(); - if (info.denom == 0) - mach_timebase_info(&info); - - result = mach_absolute_time() * (info.numer / info.denom); -#endif - -#ifdef USING_GETTIMEOFDAY - struct timeval tp; - - init_timer(); - gettimeofday( &tp, NULL ); - - result = tv_diff_us(tv_init, tp); - result = result*(uint64_t)1000; -#endif - - - return result; -} - -/** - * @brief Return the time in seconds. - * - */ -double trios_get_time() -{ - double result = 0.0; - uint64_t ns = trios_get_time_ns(); - result = (double)ns*1.0e-9; - return result; -} - -long trios_get_time_us() -{ - uint64_t result; - result = trios_get_time_ns() * 1e-3; - return result; -} - -long trios_get_time_ms() -{ - uint64_t result; - - result = trios_get_time_ns() * 1e-6; - return result; -} - -/** - * @brief Test the timer. - * - * An easy way to test the timer is to time the sleep function - * for a fixed amount of time. If the timer results a time that - * matches the sleep time within a reasonable error, the timer works. - * - * @return 0 on success, 1 on error. - */ -int trios_timer_test() -{ - int rc = 0; -/* static const uint64_t sleep_us = 1000000; */ /* 1 second */ - static const unsigned int sleep_s = 1; /* 1 second */ - - /* TODO: what should be the acceptable error (1/2 sec?) */ - static const uint64_t error_ms = 500; - - uint64_t error_ns = error_ms * 1000000; - uint64_t error_us = error_ms * 1000; - double error_sec = (double)error_ms/1000.0; - - uint64_t start_ns=0, t_ns=0; - uint64_t start_us, t_us; - uint64_t start_ms, t_ms; - double start_sec, t_sec; - - log_debug(timer_debug_level, "this is a timer test. expect a 1 second delay. starting..."); - - /* run to initialize timer */ - start_ns = trios_get_time_ns(); - if (sleep(sleep_s)) { - log_error(timer_debug_level, "Failed calling sleep()"); - return 1; - } - - /* run to get measurements */ - start_ns = trios_get_time_ns(); - start_us = trios_get_time_us(); - start_ms = trios_get_time_ms(); - start_sec = trios_get_time(); - - /* sleep one seconds */ - sleep(sleep_s); - - t_ns = trios_get_time_ns() - start_ns; - t_us = trios_get_time_us() - start_us; - t_ms = trios_get_time_ms() - start_ms; - t_sec = trios_get_time() - start_sec; - - printf("slept for %u seconds:\n", sleep_s); - printf("\tns = %lu\n", t_ns); - printf("\tus = %lu\n", t_us); - printf("\tms = %lu\n", t_ms); - printf("\tsec = %f\n", t_sec); - - /* Make sure our values have a reasonable error */ - if (labs(t_ns - ((double)sleep_s * 1e9)) > error_ns) { - uint64_t actual_err = labs(t_ns - ((double)sleep_s * 1e9)); - log_error(timer_debug_level, - "trios_timer failed ns timer test: err = %llu ns" - " > acceptable err = %llu", - actual_err, error_ns); - rc = 1; - } - - if (labs(t_us - ((double)sleep_s * 1e6)) > error_us) { - uint64_t actual_err = labs(t_us - ((double)sleep_s * 1e6)); - log_error(timer_debug_level, - "trios_timer failed usec timer test: err = %llu usec" - " > acceptable err = %llu", - actual_err, error_us); - rc |= 1; - } - - if (labs(t_ms - ((double)sleep_s * 1e3)) > error_ms) { - uint64_t actual_err = labs(t_ms - ((double)sleep_s * 1e3)); - log_error(timer_debug_level, - "trios_timer failed ns timer test: err = %llu ms" - " > acceptable err = %llu", - actual_err, error_ms); - rc |= 1; - } - - if (fabs(t_sec - (double)sleep_s) > error_sec) { - double actual_err = fabs(t_sec - (double)sleep_s); - log_error(timer_debug_level, - "trios_timer failed sec timer test: err = %g sec " - " > acceptable err = %g", - actual_err, error_sec); - rc |= 1; - } - - log_debug(timer_debug_level, "timer test complete"); - - return rc; -} - -const char *trios_timer_getimpl() -{ -#ifdef USING_PAPI - return "PAPI_get_real_usec()"; -#elif defined(USING_CLOCK_GETTIME) - return "clock_gettime()"; -#elif defined (USING_MACH_ABSOLUTE_TIME) - return "mach_absolute_time()"; -#elif defined(USING_GETTIMEOFDAY) - return "gettimeofday()"; -#else - return "no timer implementation"; -#endif -} diff --git a/packages/trios/libraries/support/trace/SDDF.cc b/packages/trios/libraries/support/trace/SDDF.cc deleted file mode 100644 index 872b8b41efa1..000000000000 --- a/packages/trios/libraries/support/trace/SDDF.cc +++ /dev/null @@ -1,645 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file pablo_interface.C - * - * @brief Makes calls to the Pablo SDDF library to generate - * trace files. - * - * @author Ron Oldfield (raoldfi@sandia.gov) - * @version $Revision: 406 $ - * @date $Date: 2005-10-07 15:08:29 -0600 (Fri, 07 Oct 2005) $ - * - */ - -#include "Trios_config.h" - -#ifdef HAVE_TRIOS_PABLO - -#include "SDDF.h" -#include -#include -#include -#include -#include - -#include -#include -#include -#include "Trios_timer.h" -#include "Trios_logger.h" -#include "Trios_threads.h" - - -/*------------ SUPPORT FOR INTERVAL EVENTS WITH PABLO ------------*/ - -/** - * A generic event has the following fields - * - * double timestamp; - * int id; - * int pid; - * char *data; - */ - - -int SDDF::define_generic_event( - const int tag, - PipeWriter *writer) -{ - Attributes *attributes = new Attributes(); - FieldDescriptor *fieldP; - StructureDescriptor *structureP; - - attributes->clearEntries(); - attributes->insert("event", "Generic trace event"); - structureP = new StructureDescriptor("event", *attributes); - - /* double timestamp */ - attributes->clearEntries(); - attributes->insert("timestamp", "Time of event"); - attributes->insert("units", "sec"); - fieldP = new FieldDescriptor("timestamp", *attributes, DOUBLE, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int id */ - attributes->clearEntries(); - attributes->insert("id", "Identifier"); - fieldP = new FieldDescriptor("id", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int pid */ - attributes->clearEntries(); - attributes->insert("pid", "Process/Thread identifier"); - fieldP = new FieldDescriptor("pid", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* char *data */ - attributes->clearEntries(); - attributes->insert("data", "App-specific character data"); - fieldP = new FieldDescriptor("data", *attributes, CHARACTER, 1); - structureP->insert(*fieldP); - delete fieldP; - - /* Now we can write the structure descriptor to the output pipe - * and create the RecordDossier for a generic event. */ - writer->putDescriptor(*structureP, tag); - - /* create the global genericRecord structure */ - genericRecord = new RecordDossier(tag, *structureP); - delete structureP; - delete attributes; - - return 0; -} - - -/** - * A count event has is a generic event with an extra count field. - * - * double timestamp; - * long count; - * int id; - * int pid; - * char *data; - */ - -int SDDF::define_count_event( - const int tag, - PipeWriter *writer) -{ - Attributes *attributes = new Attributes(); - FieldDescriptor *fieldP; - StructureDescriptor *structureP; - - attributes->clearEntries(); - attributes->insert("count", "Generic trace event"); - structureP = new StructureDescriptor("count", *attributes); - - /* double timestamp */ - attributes->clearEntries(); - attributes->insert("timestamp", "Time of event"); - attributes->insert("units", "sec"); - fieldP = new FieldDescriptor("timestamp", *attributes, DOUBLE, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* long count */ - attributes->clearEntries(); - attributes->insert("count", "Count"); - fieldP = new FieldDescriptor("count", *attributes, LONG, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int id */ - attributes->clearEntries(); - attributes->insert("id", "Identifier"); - fieldP = new FieldDescriptor("id", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int pid */ - attributes->clearEntries(); - attributes->insert("pid", "Process/Thread identifier"); - fieldP = new FieldDescriptor("pid", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* char *data */ - attributes->clearEntries(); - attributes->insert("data", "App-specific character data"); - fieldP = new FieldDescriptor("data", *attributes, CHARACTER, 1); - structureP->insert(*fieldP); - delete fieldP; - - /* Now we can write the structure descriptor to the output pipe - * and create the RecordDossier for a generic event. */ - writer->putDescriptor(*structureP, tag); - - /* create the global countRecord structure */ - countRecord = new RecordDossier(tag, *structureP); - - delete structureP; - delete attributes; - - return 0; -} - - -/** - * A count event has is a generic event with an extra count field. - * - * double timestamp; - * int id; - * int pid; - * int level; - * double duration; - * char *data; - */ -int SDDF::define_interval_event( - const int tag, - PipeWriter *writer) -{ - Attributes *attributes = new Attributes(); - FieldDescriptor *fieldP; - StructureDescriptor *structureP; - - attributes->clearEntries(); - attributes->insert("interval", "Interval event"); - structureP = new StructureDescriptor("interval", *attributes); - - /* double timestamp */ - attributes->clearEntries(); - attributes->insert("timestamp", "Time of event"); - attributes->insert("units", "sec"); - fieldP = new FieldDescriptor("timestamp", *attributes, DOUBLE, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int id */ - attributes->clearEntries(); - attributes->insert("id", "Identifier"); - fieldP = new FieldDescriptor("id", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int pid */ - attributes->clearEntries(); - attributes->insert("pid", "Process/Thread identifier"); - fieldP = new FieldDescriptor("pid", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int level */ - attributes->clearEntries(); - attributes->insert("level", "Level/Scope"); - fieldP = new FieldDescriptor("level", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* double duration */ - attributes->clearEntries(); - attributes->insert("duration", "Length of interval"); - attributes->insert("units", "sec"); - fieldP = new FieldDescriptor("duration", *attributes, DOUBLE, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* char *data */ - attributes->clearEntries(); - attributes->insert("data", "App-specific character data"); - fieldP = new FieldDescriptor("data", *attributes, CHARACTER, 1); - structureP->insert(*fieldP); - delete fieldP; - - /* Now we can write the structure descriptor to the output pipe - * and create the RecordDossier for a generic event. */ - writer->putDescriptor(*structureP, tag); - - /* create the global intervalRecord structure */ - intervalRecord = new RecordDossier(tag, *structureP); - - delete structureP; - delete attributes; - - return 0; -} - - -/** - * A throughput event has the following fields - * - * double timestamp; - * int id; - * int pid; - * int level; - * char *data; - * double duration; - * long count; - */ - -int SDDF::define_throughput_event( - const int tag, - PipeWriter *writer) -{ - Attributes *attributes = new Attributes(); - FieldDescriptor *fieldP; - StructureDescriptor *structureP; - - attributes->clearEntries(); - attributes->insert("throughput", "Throughput Event"); - structureP = new StructureDescriptor("throughput", *attributes); - - /* double timestamp */ - attributes->clearEntries(); - attributes->insert("timestamp", "Time of event"); - attributes->insert("units", "sec"); - fieldP = new FieldDescriptor("timestamp", *attributes, DOUBLE, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int id */ - attributes->clearEntries(); - attributes->insert("id", "Identifier"); - fieldP = new FieldDescriptor("id", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int pid */ - attributes->clearEntries(); - attributes->insert("pid", "Process/Thread identifier"); - fieldP = new FieldDescriptor("pid", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* int level */ - attributes->clearEntries(); - attributes->insert("level", "Level/Scope"); - fieldP = new FieldDescriptor("level", *attributes, INTEGER, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* double duration */ - attributes->clearEntries(); - attributes->insert("duration", "Length of interval"); - attributes->insert("units", "sec"); - fieldP = new FieldDescriptor("duration", *attributes, DOUBLE, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* long count */ - attributes->clearEntries(); - attributes->insert("count", "Number of objects processed"); - fieldP = new FieldDescriptor("count", *attributes, LONG, 0); - structureP->insert(*fieldP); - delete fieldP; - - /* char *data */ - attributes->clearEntries(); - attributes->insert("data", "App-specific character data"); - fieldP = new FieldDescriptor("data", *attributes, CHARACTER, 1); - structureP->insert(*fieldP); - delete fieldP; - - /* Now we can write the structure descriptor to the output pipe - * and create the RecordDossier for a generic event. */ - writer->putDescriptor(*structureP, tag); - - /* create the global throughputRecord structure */ - throughputRecord = new RecordDossier(tag, *structureP); - - delete structureP; - delete attributes; - - return 0; -} - - -/** - * @brief Initialize the pablo output interface. - * - * @param fname @input_type The name of the outputfile. - * @param type @input_type Type of file (0=binary, 1=ascii) - * - */ -SDDF::SDDF(const char *f, const int t): - TraceFile(f), ftype(t), genericRecord(0), countRecord(0), - intervalRecord(0), throughputRecord(0) -{ - static const int bufsize = 204800; - - /* get the current time */ - time_t now = time(NULL); - - /* store the start time */ - starttime = trios_get_time(); - - Attributes attributes; - - - /* initialize the mutex for the genericRecord */ - nthread_lock_init(&genericMutex); - /* initialize the mutex for the countRecord */ - nthread_lock_init(&countMutex); - /* initialize the mutex for the intervalRecord */ - nthread_lock_init(&intervalMutex); - /* initialize the mutex for the throughputRecord */ - nthread_lock_init(&throughputMutex); - - /* initialize the mutex for the output stream */ - nthread_lock_init(&outputMutex); - - - /* Open file */ - outFile = new OutputFileStreamPipe(fname, bufsize); - if (ftype) { - pipeWriter = new AsciiPipeWriter(outFile); - } - else { - pipeWriter = new BinaryPipeWriter(outFile); - } - - /* Stream Attribute */ - attributes.clearEntries(); - attributes.insert("run date", ctime(&now)); - /* ... what else goes in the header? */ - pipeWriter->putAttributes(attributes); - - //output_header(pipeWriter); - - /* ---- Describe the types of records we expect ---- */ - define_generic_event(GENERIC_RECORD, pipeWriter); - define_count_event(COUNT_RECORD, pipeWriter); - define_interval_event(INTERVAL_RECORD, pipeWriter); - define_throughput_event(THROUGHPUT_RECORD, pipeWriter); - - output_generic_event(0,0,"init"); - output_count_event(0,0,"init",0); - output_interval_event(0,0,0,"init",0); - output_tput_event(0,0,0,"init",0,0); -} - -SDDF::~SDDF(void) -{ - fprintf(stderr, "delete outfile\n"); - delete outFile; - fprintf(stderr, "delete pipewriter\n"); - delete pipeWriter; - fprintf(stderr, "delete genericRec\n"); - delete genericRecord; - fprintf(stderr, "delete countRecord\n"); - delete countRecord; - fprintf(stderr, "delete throughputRecord\n"); - delete throughputRecord; - fprintf(stderr, "delete intervalRecord\n"); - delete intervalRecord; - - /* cleanup the mutex for the genericRecord */ - nthread_lock_fini(&genericMutex); - /* cleanup the mutex for the countRecord */ - nthread_lock_fini(&countMutex); - /* cleanup the mutex for the intervalRecord */ - nthread_lock_fini(&intervalMutex); - /* cleanup the mutex for the throughputRecord */ - nthread_lock_fini(&throughputMutex); - - /* cleanup the mutex for the output stream */ - nthread_lock_fini(&outputMutex); - - - fprintf(stderr, "finished ~SDDF()\n"); -} - -int SDDF::output_record(RecordDossier *rec) -{ - int rc = 0; - - if (!rec) - return rc; - /* need to protect the putData function */ - nthread_lock(&mutex); - pipeWriter->putData(*rec); - nthread_unlock(&mutex); - - return rc; -} - -/** - * @brief Output a generic trace event. - * - * @param eventID @input_type The ID of the trace. - * @param pid @input_type Process ID. - * @param data @input_type User-defined data passed in a character string. - * - * @return non-zero if successfull - * @return 0 if failure - */ -int SDDF::output_generic_event( - const int eventID, - const int pid, - const char *data) -{ - int rc = 1; - - nthread_lock(&genericMutex); - - genericRecord->setValue("timestamp", trios_get_time() - starttime); - genericRecord->setValue("id", eventID); - genericRecord->setValue("pid", pid); - - /*fprintf(stderr, "setting data to \"%s\"\n",data);*/ - genericRecord->setCString("data", data); - - nthread_unlock(&genericMutex); - - output_record(genericRecord); - - return rc; -} - - - -/** - * @brief Output an interval event. - * - * We use the generic pablo trace and encode the additional information - * we want in the data field. The new data field will be, "interval:$name:duration". - * - * Pablo has its own interval records, but they are inadequate because it is - * difficult to measure concurrent intervals (e.g., in threads). - * A better (and more efficient) way to do this would be to create our own - * Pablo record type, but this is a quick "hack" to address our needs. - * - * @param eventID @input_type The ID of the trace. - * @param pid @input_type Process ID. - * @param level @input_type - * @param data @input_type User-defined data passed in a character - * string. - * - * @return non-zero if successfull - * @return 0 if failure - */ -int SDDF::output_interval_event( - const int eventID, - const int pid, - const int level, - const char *data, - double duration) -{ - int rc = 1; - - nthread_lock(&intervalMutex); - intervalRecord->setValue("timestamp", trios_get_time() - starttime); - intervalRecord->setValue("id", eventID); - intervalRecord->setValue("pid", pid); - intervalRecord->setValue("level", level); - intervalRecord->setValue("duration", duration); - - /*fprintf(stderr, "setting data to \"%s\"\n",data);*/ - intervalRecord->setCString("data", data); - nthread_unlock(&intervalMutex); - - output_record(intervalRecord); - - return rc; -} - -int SDDF::output_tput_event( - const int eventID, - const int pid, - const int level, - const char *data, - double duration, - const long count) -{ - int rc = 1; - - nthread_lock(&throughputMutex); - throughputRecord->setValue("timestamp", trios_get_time() - starttime); - throughputRecord->setValue("id", eventID); - throughputRecord->setValue("pid", pid); - throughputRecord->setValue("level", level); - throughputRecord->setValue("duration", duration); - throughputRecord->setValue("count", count); - nthread_unlock(&throughputMutex); - - /*fprintf(stderr, "setting data to \"%s\"\n",data);*/ - throughputRecord->setCString("data", data); - - output_record(throughputRecord); - - return rc; -} - - -/** - * @brief Output a count event. - * - * We use the generic pablo trace and encode the additional information - * we want in the data field. The new data field will be, "interval:$name:duration". - * - * Pablo has its own count records, but they are inadequate because they only - * increment values. We want to increment, decrement, and set count events. - * A better (and more efficient) way to do this would be to create our own - * Pablo record type, but this is a quick "hack" that will still work. - * - * @param intervalID @input_type The interval ID (unique for each interval). - * @param eventID @input_type The ID of the trace. - * @param data @input_type User-defined data passed in a character - * string. - * - * @return non-zero if successfull - * @return 0 if failure - */ -int SDDF::output_count_event( - const int eventID, - const int pid, - const char *data, - const int count) -{ - int rc = 1; - - nthread_lock(&countMutex); - countRecord->setValue("timestamp", trios_get_time() - starttime); - countRecord->setValue("id", eventID); - countRecord->setValue("pid", pid); - countRecord->setValue("count", count); - - /*fprintf(stdout, "setting count data to \"%s\"\n",data);*/ - countRecord->setCString("data", data); - nthread_unlock(&countMutex); - - output_record(countRecord); - - return rc; -} - -int SDDF::set_buffer_size(const unsigned long int bufsize) -{ - return -1; - -} - -#endif // HAVE_TRIOS_PABLO diff --git a/packages/trios/libraries/support/trace/SDDF.h b/packages/trios/libraries/support/trace/SDDF.h deleted file mode 100644 index 3231d2babf1b..000000000000 --- a/packages/trios/libraries/support/trace/SDDF.h +++ /dev/null @@ -1,156 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#ifndef _SDDF_H_ -#define _SDDF_H_ - -#include "Trios_config.h" - -#ifdef HAVE_TRIOS_PABLO - -#include "TraceFile.h" -#include -#include -#include -#include - -#include "Trios_threads.h" - - -/** - * @brief A class for outputting SDDF trace files - * using Pablo. - */ -class SDDF : public TraceFile { - - private: - int ftype; - - enum RecordTags { - GENERIC_RECORD=1, - COUNT_RECORD, - INTERVAL_RECORD, - THROUGHPUT_RECORD - }; - - /* The RecordDossier holds information about the - * structure of a record. */ - RecordDossier *genericRecord; - RecordDossier *countRecord; - RecordDossier *intervalRecord; - RecordDossier *throughputRecord; - - /* need mutexes to protect the RecordDossier */ - nthread_lock_t genericMutex; - nthread_lock_t countMutex; - nthread_lock_t intervalMutex; - nthread_lock_t throughputMutex; - - /* Where to put the data */ - OutputFileStreamPipe *outFile; - PipeWriter *pipeWriter; - - /* need mutex to protect the output stream */ - nthread_lock_t outputMutex; - - /* When the trace file is initialized */ - double starttime; - - public: - SDDF(const char *f, const int t); - - ~SDDF(); - - int output_record(RecordDossier *rec); - - int output_generic_event( - const int eventID, - const int pid, - const char *data); - - int output_interval_event( - const int eventID, - const int pid, - const int level, - const char *data, - double duration); - - int output_tput_event( - const int eventID, - const int pid, - const int level, - const char *data, - double duration, - const long num_processed); - - int output_count_event( - const int eventID, - const int pid, - const char *data, - const int count); - - int set_buffer_size( - const unsigned long int bufsize); - - private: - int define_generic_event( - const int tag, - PipeWriter *pipeWriter); - - int define_count_event( - const int tag, - PipeWriter *pipeWriter); - - int define_interval_event( - const int tag, - PipeWriter *pipeWriter); - - int define_throughput_event( - const int tag, - PipeWriter *pipeWriter); - -}; - - -#endif /* HAVE_TRIOS_PABLO */ - -#endif /* _SDDF_H_*/ diff --git a/packages/trios/libraries/support/trace/TraceFile.h b/packages/trios/libraries/support/trace/TraceFile.h deleted file mode 100644 index 2ed521d6cde9..000000000000 --- a/packages/trios/libraries/support/trace/TraceFile.h +++ /dev/null @@ -1,97 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#ifndef TRACE_FILE_H_ -#define TRACE_FILE_H_ - - -/** - * @brief An abstract base class for trace files. - */ -class TraceFile { - - public: - const char *fname; - - public: - TraceFile(const char *f): fname(f) { } - - virtual ~TraceFile() {} - - /** - * @brief Return the file name. - */ - const char *get_fname() {return fname;} - - virtual int output_generic_event( - const int eventID, - const int pid, - const char *data) = 0; - - virtual int output_interval_event( - const int eventID, - const int pid, - const int level, - const char *data, - double duration) = 0; - - virtual int output_tput_event( - const int eventID, - const int pid, - const int level, - const char *data, - double duration, - const long num_processed) = 0; - - virtual int output_count_event( - const int eventID, - const int pid, - const char *data, - const int count) = 0; - - virtual int set_buffer_size( - const unsigned long int bufsize) = 0; -}; - - - -#endif /*TRACE_FILE_H*/ diff --git a/packages/trios/libraries/support/trace/TraceGroup.h b/packages/trios/libraries/support/trace/TraceGroup.h deleted file mode 100644 index b3d0c0b5cf91..000000000000 --- a/packages/trios/libraries/support/trace/TraceGroup.h +++ /dev/null @@ -1,130 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#ifndef TRACE_GROUP_H_ -#define TRACE_GROUP_H_ - -#include -#include - -using namespace std; -using namespace __gnu_cxx; - - - - -/** - * @brief The data structure used to manage trace groups. - */ -class TraceGroup { - - public: - TraceGroup *parent; - const char *name; - const int gid; - bool enabled; - - /* Children of this group */ - list children; - - public: - TraceGroup(const char *s, const int id, bool enable_flag): - parent(0), name(s), gid(id), enabled(enable_flag), children() { } - - ~TraceGroup() {} - - /** - * @brief Set the enabled flag on this group and all its children. - */ - int enable() { - enabled = true; - - /* enable each of the children */ - //for_each(children.begin(), children.end(), enable_group); - - return 0; - } - - /** - * @brief Enable the specified TraceGroup. - */ - static void enable_group(TraceGroup *grp) { - grp->enable(); - } - - static void disable_group(TraceGroup *g) { - g->disable(); - } - - struct disabler { - void operator() (TraceGroup *g) { - g->disable(); - } - }; - - /** - * @brief Set the disabled flag on this group and all its children. - */ - int disable() { - enabled = false; - - /* enable each of the children */ - for_each(children.begin(), children.end(), disabler()); - return 0; - } - - - /** - * @brief Add a sub-group to this group. - */ - int add_child(TraceGroup *child) { - children.push_back(child); - if (!child->parent) { - child->parent = this; - } - return 0; - } -}; - - - -#endif /*TRACE_GROUP_H*/ diff --git a/packages/trios/libraries/support/trace/Trios_trace.h b/packages/trios/libraries/support/trace/Trios_trace.h deleted file mode 100644 index abcb0aab5a4b..000000000000 --- a/packages/trios/libraries/support/trace/Trios_trace.h +++ /dev/null @@ -1,368 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file Trios_trace.h - * - * @brief A simple tracing API. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 406 $. - * $Date: 2005-10-07 15:08:29 -0600 (Fri, 07 Oct 2005) $. - * - */ - -#ifndef _TRACE_H_ -#define _TRACE_H_ - -#include - -#ifdef __cplusplus -extern "C" { -#endif - - enum trace_ftype { - TRACE_UNDEFINED_FTYPE = -1, - TRACE_SDDF_BINARY, - TRACE_SDDF_ASCII - }; - -#define TRACE_DEFAULT_FTYPE TRACE_SDDF_BINARY; - -#if defined(__STDC__) || defined(__cplusplus) - - /* initialization and wrapup functions */ - extern int trace_init(const char *trace_fname, const int trace_ftype); - extern int trace_reset( - const char *trace_fname, - const int trace_ftype, - const char *enable); - extern int trace_set_buffer_size(const unsigned long int bufsize); - extern int trace_set_buffer_size(const unsigned long int bufsize); - /* extern int trace_set_processor_number(const int pnum); */ - extern int trace_fini(void); - - - /* ----------- Trace Group Functions --------------- */ - - /** - * @brief Register a new top-level group for tracing. - * - * @param name The name of the group. - * @param gid The groupid assigned by the trace library. - * - * @return 0 on success, -1 otherwise. - */ - extern int trace_register_group(const char *name, int *gid); - - /** - * @brief Register a trace group as a child of an existing - * group. - * - * @param name The name of the new group. - * @param parent_id The group ID of the parent group. - * @param gid The value of this group. - */ - extern int trace_register_subgroup(const char *name, int parent_id, int *gid); - - /** - * @brief Return the group id for a particular named group. - * - * @param name The name of the group to find. - * - * @returns The group ID the group, if found; -1 otherwise. - */ - extern int trace_get_gid(const char *name); - - /** - * @brief Return the name of an existing group. - * - * @param gid The group ID of the group to find. - * - * @returns The name of the group, if found; NULL otherwise. - */ - extern const char *trace_get_gname(const int gid); - - - /** - * @brief Test to see if tracing is enabled. - * - * @param gid The groupID to check - */ - extern int tracing_enabled(const int gid); - - /** - * @brief Enable all groups listed in a comma-separated - * list of group names. - * - * @param csl The comma-separated list of groups to enable - */ - extern int trace_enable(const char *csl); - - /** - * @brief Enable the specified trace group and all its children. - * - * @param gid The group ID. - * - * @return 0 on success, -1 if the gid is invalid. - */ - extern int trace_enable_gid(const int gid); - - /** - * @brief Disable a specified trace group and all its children. - * - * @param gid The group ID of the trace group to disable. - * - * @return 0 on success, -1 if the gid is invalid. - */ - extern int trace_disable(const int gid); - - /** - * @brief Enable every registered trace group. - * - * This function enables all trace groups, thus turning on - * all tracing. It also sets the default enabled flag to - * true so that any group registered after this call is - * also enabled. - */ - extern int trace_enable_all(); - - /** - * @brief Disable every registered trace group. - * - * This function disables all trace groups, thus disabling - * all tracing. It also sets the default enabled flag to - * false so that any group registered after this call is - * also disabled. - */ - extern int trace_disable_all(); - - /* ----------------- Trace Event Functions ------------ */ - - - /** - * @brief Generate a generic trace event. - * - * @param gid The trace group ID. - * @param eventID The ID of the trace. - * @param pid Process ID. - * @param data User-defined data passed in a character string. - * - * @return non-zero if successful - * @return 0 if failure - */ - extern int trace_event( - const int gid, - const int eventID, - const int pid, - const char *data); - - /* interval functions */ - - - - /** - * @brief Generate an interval start event. - * - * The \ref trace_start_interval "trace_start_interval()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * We use the generic pablo trace and encode the additional information - * we want in the data field. The new data field will be, "interval:$name:duration". - * - * Pablo has its own interval records, but they are inadequate because it is - * difficult to measure concurrent intervals (e.g., in threads). - * A better (and more efficient) way to do this would be to create our own - * Pablo record type, but this is a quick "hack" to address our needs. - * - * @param gid The trace group ID. - * @param pid The ID of process/thread. - * - * @return non-zero if successful - * @return 0 if failure - */ - extern int trace_start_interval( - const int gid, - const int pid); - - /** - * @brief Generate an end-interval event. - * - * The \ref trace_end_interval "trace_end_interval()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * @param gid The trace group ID. - * @param event_id The ID of the trace (could be the thread ID). - * @param pid Process ID. - * @param data User-defined data passed in a character - * string. - * - * @return non-zero if successful - * @return 0 if failure - */ - extern int trace_end_interval( - const int gid, - const int event_id, - const int pid, - const char *data); - - - /** - * @brief Generate a throughput interval start event. - * - * The \ref trace_start_tput_interval "trace_start_tput_interval()" function generates an - * throughput interval record that has information about the start, end, duration, and - * number of elements processed during the interval. It is useful for measuring - * bandwidth, ops/sec, ... - * - * We use the generic pablo trace and encode the additional information - * we want in the data field. The new data field will be, "interval:tput:$name:duration". - * - * Pablo has its own interval records, but they are inadequate because it is - * difficult to measure concurrent intervals (e.g., in threads). - * A better (and more efficient) way to do this would be to create our own - * Pablo record type, but this is a quick "hack" to address our needs. - * - * @param interval_id The ID of the interval (unique for each interval) - * @param pid The ID of the process/thread - * - * @return non-zero if successful - * @return 0 if failure - */ - extern int trace_start_tput_interval(const int gid, const int pid); - - /** - * @brief Generate an end-interval event. - * - * The \ref trace_end_interval "trace_end_interval()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * @param gid The trace group ID. - * @param event_id The ID of the trace (could be the thread ID). - * @param pid Process ID. - * @param data User-defined data passed in a character - * string. - * - * @return non-zero if successful. - * @return 0 if failure - */ - extern int trace_end_tput_interval( - const int gid, - const int event_id, - const int pid, - const long num_processed, - const char *data); - - - /* extern int trace_reset_interval(const int eventID); */ - - /* count events */ - - /** - * @brief Get the current value of a trace counter. - * - * @param gid The trace group ID. - * @param cid The ID of the counter. - * - * @return Value of the counter. - */ - extern int trace_get_count( - const int gid, - const int cid); - - /** - * @brief Generate an increment-count event. - * - * The trace_inc_count_event() function - * increments a counter associated with the ID of counter and outputs - * a count event to the tracefile. - * - * @param gid The trace group ID. - * @param cid The ID of the counter. - * @param pid Process/thread ID. - * @param data User-defined data in a character string. - * - * @returns non-zero if successful. - * @returns 0 if failure. - */ - extern int trace_inc_count( - const int gid, - const int cid, - const int pid, - const char *data); - - - extern int trace_dec_count( - const int gid, - const int cid, - const int pid, - const char *data); - - extern int trace_set_count( - const int gid, - const int cid, - const int pid, - const char *data, - const int newval); - - extern int trace_reset_count( - const int gid, - const int cid, - const int pid, - const char *data); - - extern int trace_put_all_counts( - const int gid, - const int pid, - const char *data); - - -#endif /* defined(__STDC__) || defined(__cplusplus) */ - - -#ifdef __cplusplus -} -#endif - -#endif /* _TRACE_H_ */ diff --git a/packages/trios/libraries/support/trace/trace.cc b/packages/trios/libraries/support/trace/trace.cc deleted file mode 100644 index 63efe93e68c2..000000000000 --- a/packages/trios/libraries/support/trace/trace.cc +++ /dev/null @@ -1,1094 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file trace.c - * - * @brief A simple library for generating timing traces. - * - * @author Ron Oldfield (raoldfi@sandia.gov) - * @version $Revision: 406 $ - * @date $Date: 2005-10-07 15:08:29 -0600 (Fri, 07 Oct 2005) $ - * - */ - -#include "Trios_config.h" - -#include "Trios_trace.h" - - -#include "TraceFile.h" - -#ifndef TRIOS_ENABLE_TRACING -#include "trace_noop.c" -#else - -#ifdef HAVE_TRIOS_PABLO -#include "SDDF.h" -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include "Trios_trace.h" -#include "Trios_timer.h" -#include "Trios_logger.h" -#include "Trios_threads.h" - -#include "TraceGroup.h" - -#include -#include - -/* The tracefile class */ -static TraceFile *tracefile = 0; - -/* hash tables for managing the trace groups */ -static std::map gname_map; -typedef std::map::iterator gname_iterator_t; -typedef std::pair gname_pair_t; -static nthread_lock_t gname_mutex; - -static std::map gid_map; -typedef std::map::iterator gid_iterator_t; -typedef std::pair gid_pair_t; -static nthread_lock_t gid_mutex; - -/* --- Hashtable that stores the intervals --- */ -static std::map > interval_map; -typedef std::map >::iterator interval_iterator_t; -typedef std::pair > interval_pair_t; -static nthread_lock_t interval_mutex; - -/* --- Hashtable to store counter information --- */ -static std::map count_map; -typedef std::map::iterator count_iterator_t; -typedef std::pair count_pair_t; - -static nthread_lock_t count_mutex; - - -static bool default_enable_flag = false; - -#ifndef HAVE_TRIOS_PABLO -#warning Building trace library without Pablo -#endif - - -log_level trace_debug_level = LOG_UNDEFINED; - -/* ------------------ PUBLIC API ----------------------- */ - - -/*------------ Initialization and wrapup functions ------------*/ - - - -int trace_set_buffer_size(const unsigned long int bufsize) -{ - int rc = 1; - if (tracefile) { - rc = tracefile->set_buffer_size(bufsize); - } - return rc; -} - - -/** - * @brief Register a new top-level group for tracing. - */ - -int trace_register_group(const char *name, int *gid) -{ - static volatile int groupcount = 0; - gname_iterator_t name_iter; - - /* Find out of the group already exists. If so, return gid */ - nthread_lock(&gname_mutex); - if ((name_iter = gname_map.find(name)) != gname_map.end()) { - *gid = name_iter->second->gid; - nthread_unlock(&gname_mutex); - return 0; - } - - /* increment the groupcount and assign the id of this new group */ - *gid = ++groupcount; - -// fprintf(logger_get_file(), "registering trace group %s (gid=%d)\n", -// name, *gid); - - /* create the new group */ - TraceGroup *group = new TraceGroup(name, *gid, default_enable_flag); - - /* Add the group to the gname_map and the gid_map */ - gname_map.insert(pair(group->name, group)); - nthread_unlock(&gname_mutex); - - - nthread_lock(&gid_mutex); - gid_map.insert(pair(group->gid, group)); - nthread_unlock(&gid_mutex); - - return 0; -} - -/** - * @brief Register a trace group as a child of an existing - * group. - * - * @param name The name of the new group. - * @param parent_id The group ID of the parent group. - * @param gid The value of this group. - */ -int trace_register_subgroup( - const char *name, - int parent_id, - int *gid) -{ - int rc; - gid_iterator_t parent_iter, child_iter; - TraceGroup *child = NULL; - - rc = trace_register_group(name, gid); - - nthread_lock(&gid_mutex); - child_iter = gid_map.find(*gid); - child = child_iter->second; - - /* if the partent group exists, add this node as a child */ - parent_iter = gid_map.find(parent_id); - if (parent_iter != gid_map.end()) { - parent_iter->second->add_child(child); - } - nthread_unlock(&gid_mutex); - - return rc; -} - - -/** - * @brief Return the group id for a particular named group. - * - * @param name The name of the group to find. - * - * @returns The group ID the group, if found; -1 otherwise. - */ - -int trace_get_gid(const char *name) -{ - int gid; - gname_iterator_t name_iter; - - /* lookup the group by calling trace_register. If the - * group is already registered it just returns the gid - */ - trace_register_group(name, &gid); - - return gid; -} - -/** - * @brief Return true if the group is enabled. - * - * @param gid The groupID to check - */ - -int tracing_enabled(const int gid) -{ - int result = false; - - /* first check that they initialized the trace environment */ - if (tracefile) { - - /* find the gid */ - nthread_lock(&gid_mutex); - gid_iterator_t gid_iter = gid_map.find(gid); - - if (gid_iter == gid_map.end()) { - result = false; - } - else { - result = gid_iter->second->enabled; - } - nthread_unlock(&gid_mutex); - } - - return result; -} - -/** - * @brief Return the name of an existing group. - * - * @param gid The group ID of the group to find. - * - * @returns The name of the group, if found; NULL otherwise. - */ - -const char *trace_get_gname(const int gid) -{ - gid_iterator_t gid_iter; - - /* lookup the group by gid */ - gid_iter = gid_map.find(gid); - - if (gid_iter == gid_map.end()) { - return 0; - } - else { - return gid_iter->second->name; - } -} - - - -/** - * Enable a single TraceGroup that is part of a pair. - */ -static void enable_group(pair< int, TraceGroup *> node) -{ - node.second->enabled = true; -} - - -/** - * @brief Enable all groups listed in a comma-separated - * list of group names. - * - * @param csl The comma-separated list of groups to enable - * - * @return 0 if successfull, -1 on error. - */ -int trace_enable(const char *csl) -{ - int rc = 0; - - if (csl) { - - const char delim[] = ","; - char *token; - char *saveptr; - - token = strtok_r((char *)csl, delim, &saveptr); - while (token != NULL) { - int gid; - - if (strcasecmp(token, "all") == 0) { - fprintf(logger_get_file(), - "enabling all registered trace groups\n" ); - trace_enable_all(); - } - else { - gid = trace_get_gid(token); - fprintf(logger_get_file(), "enabling trace " - "group %s (gid=%d)\n", - token, gid); - trace_enable_gid(gid); - } - - token = strtok_r(NULL, delim, &saveptr); - } - } - else { - fprintf(logger_get_file(),"tracing disabled\n"); - } - - return rc; -} - - -/** - * @brief Set the enable flag for a group and all its children. - * - * @param gid The group ID. - * - * @return 0 on success, -1 if the groupID is invalid. - */ -int trace_enable_gid(const int gid) -{ - int result = 0; - gid_iterator_t gid_iter; - - /* lookup the group by its gid */ - nthread_lock(&gid_mutex); - gid_iter = gid_map.find(gid); - - /* case for bad gid */ - if (gid_iter == gid_map.end()) { - result = -1; - } - else { - enable_group(*gid_iter); - } - nthread_unlock(&gid_mutex); - - - return result; -} - - -/** - * @brief Enable every registered trace group. - */ -int trace_enable_all() -{ - gid_iterator_t iter; - /* Disable each trace group in the gid map */ - nthread_lock(&gid_mutex); - - for (iter=gid_map.begin(); iter!=gid_map.end(); iter++) { - iter->second->enabled = true; - } - - //for_each(gid_map.begin(), gid_map.end(), enable_group); - - /* set the default enable flag to false (for future groups) */ - default_enable_flag = true; - nthread_unlock(&gid_mutex); - - return 0; -} - - - -/** - * @brief Disable a trace group. - * - * @param gid The group ID of the trace group to disable. - * - * @return 0 on success, -1 if the gid is invalid. - */ -int trace_disable(const int gid) -{ - gid_iterator_t gid_iter; - int result = 0; - - /* lookup the group by its gid */ - nthread_lock(&gid_mutex); - gid_iter = gid_map.find(gid); - - /* case for bad gid */ - if (gid_iter == gid_map.end()) { - result = -1; - } - else { - /* Call the enable() function on the TraceGroup */ - gid_iter->second->disable(); - } - nthread_unlock(&gid_mutex); - - return result; -} - -/** - * Disable a single TraceGroup that is part of a pair. - */ -static void disable_group(pair node) -{ - node.second->enabled = false; -} - - -/** - * @brief Disable all trace groups. - */ -int trace_disable_all() -{ - /* Disable each trace group in the gid map */ - nthread_lock(&gid_mutex); - for_each(gid_map.begin(), gid_map.end(), disable_group); - - /* set the default enable flag to false (for future groups) */ - default_enable_flag = false; - nthread_unlock(&gid_mutex); - - return 0; -} - - - -/** - * @brief Reset the tracing API. - * - * This means we close any existing files, reset all counters, - * and open a new trace file. - */ -int trace_reset(const char *fname, const int ftype, const char *csl) -{ - int rc = 0; - count_iterator_t iter; - log_level debug_level = LOG_ALL; - - log_debug(debug_level, "trace_reset start: fname=%s, ftype=%d, enable=%s", - fname, ftype, csl); - - /* Close the trace file */ - if (tracefile) { - fprintf(logger_get_file(),"closing tracefile\n"); - delete tracefile; - tracefile = 0; - } - - /* Init the trace file */ - log_debug(trace_debug_level, "init tracing"); - rc = trace_init(fname, ftype); - if (rc != 0) { - return rc; - } - - /* Reset all counters */ - log_debug(trace_debug_level, "reset counters"); - nthread_lock(&count_mutex); - for (iter = count_map.begin(); iter != count_map.end(); iter++) { - iter->second = 0; - } - nthread_unlock(&count_mutex); - - /* Disable all trace groups */ - log_debug(trace_debug_level, "disable trace groups"); - rc = trace_disable_all(); - if (rc != 0) - return rc; - - /* Enable the groups in the csl */ - log_debug(trace_debug_level, "enable %s",csl); - rc = trace_enable(csl); - if (rc != 0) { - return rc; - } - - log_debug(trace_debug_level, "trace_reset exit"); - return rc; -} - - -/** - * @brief Initialize the tracing environment. - * - * @param fname The name of the trace file. - * @param ftype The type of tracing (currently unused). - */ -int trace_init(const char *fname, const int ftype) -{ - int rc = 0; - log_level debug_level = trace_debug_level; - - log_debug(debug_level, "trace_init(fnam=%s)",fname); - log_debug(debug_level, "trace_init(ftype=%d)",ftype); - - log_debug(trace_debug_level, "trace_init(%s,%d)",fname,ftype); - - nthread_lock_init(&gname_mutex); - nthread_lock_init(&gid_mutex); - nthread_lock_init(&interval_mutex); - nthread_lock_init(&count_mutex); - - - if (!tracefile && fname) { - int fd; - - /* make sure we can even create the tracefile */ - fd = open(fname, O_CREAT | O_RDWR); - if (fd < 0) { - log_warn(trace_debug_level, "Could not open trace file %s... tracing disabled", - fname); - /* do nothing (disable tracing) if we cannot create the file */ - return rc; - } - else { - close (fd); - remove(fname); - } - - switch (ftype) { - -#ifdef HAVE_TRIOS_PABLO - - case TRACE_SDDF_BINARY: - log_debug(trace_debug_level, "SDDF output to \"%s\"", - fname); - tracefile = new SDDF(fname, 0); - break; - - case TRACE_SDDF_ASCII: - tracefile = new SDDF(fname, 1); - break; -#endif - - default: - fprintf(stderr, "Trace file type %d not recognized\n", - ftype); - rc = -1; - } - } - - log_debug(trace_debug_level, "trace_init exit"); - - return rc; -} - -static void delete_group(gname_pair_t entry) -{ - TraceGroup *grp = entry.second; - delete grp; -} - -/** - * @brief Finalize the tracing environment. - */ -int trace_fini() -{ - int rc = 0; - - if (tracefile) { - delete tracefile; - tracefile = 0; - } - - /* delete trace groups */ - nthread_lock(&gname_mutex); - for_each(gname_map.begin(), gname_map.end(), delete_group); - - - gname_map.clear(); - nthread_unlock(&gname_mutex); - - nthread_lock(&gid_mutex); - gid_map.clear(); - nthread_unlock(&gid_mutex); - - nthread_lock(&interval_mutex); - interval_map.clear(); - nthread_unlock(&interval_mutex); - - nthread_lock_fini(&gname_mutex); - nthread_lock_fini(&gid_mutex); - nthread_lock_fini(&interval_mutex); - nthread_lock_fini(&count_mutex); - - return rc; -} - - - -/** - * @brief Generate a generic trace event. - * - * @param eventID @input_type The ID of the trace. - * @param pid @input_type Process ID. - * @param data @input_type User-defined data passed in a character string. - * - * @return non-zero if successfull - * @return 0 if failure - */ -int trace_event( - const int gid, - const int eventID, - const int pid, - const char *data) -{ - int rc = 0; - - if (tracefile && tracing_enabled(gid)) { - rc = tracefile->output_generic_event(eventID,pid,data); - } - - return rc; -} - - - - -/** - * @brief Generate an interval start event. - * - * The \ref trace_start_interval "trace_start_interval()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * Intervals are stored internally as a stack for each process ID. When the - * end_interval event arrives, we pop the starttime off the stack and use that - * to calculate the duration. Then we write an interval event to the - * underlying trace framework. - * - * Pablo has its own interval records, but they are inadequate because it is - * difficult to measure concurrent intervals (e.g., in threads). - * - * @param gid The trace group ID. - * @param interval_id @output_type The ID of the interval (unique for each interval) - * - * @return non-zero if successfull - * @return 0 if failure - */ -int trace_start_interval( - const int gid, - const int pid) -{ - int rc = 0; - - log_debug(trace_debug_level, "trace_start_interval(gid=%d) start", - pid, gid); - - if (tracefile && tracing_enabled(gid)) { - - double time = trios_get_time(); - interval_iterator_t iter; - - /* see if an interval entry for this pid already exists */ - nthread_lock(&interval_mutex); - iter = interval_map.find(pid); - - /* if the interval entry already exists, we push the starttime on the stack */ - if (iter != interval_map.end()) { - iter->second.push(time); - } - - /* register the start time of the interval */ - else { - stack s; - s.push(time); - interval_map.insert(pair< int, stack >(pid, s)); - } - nthread_unlock(&interval_mutex); - } - - log_debug(trace_debug_level, "trace_start_interval(gid=%d) end", gid); - - return rc; -} - - -/** - * @brief Generate an end-interval event. - * - * The \ref trace_interval_event "trace_interval_event()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * @param intervalID @input_type The interval ID (unique for each interval). - * @param eventID @input_type The ID of the trace. - * @param pid @input_type Process ID. - * @param level @input_type The depth of the interval (0=root, 1=sub-interval, 2=sub-sub,...). - * @param data @input_type User-defined data passed in a character - * string. - * - * @return non-zero if successfull - * @return 0 if failure - */ -int trace_end_interval( - const int gid, - const int eventID, - const int pid, - const char *data) -{ - int level = pid; - int rc; - log_level debug_level = trace_debug_level; - - log_debug(debug_level, "%d: trace_end_interval(gid=%d) start", - pid, gid); - - if (tracefile && tracing_enabled(gid)) { - double endtime = trios_get_time(); - double duration=0; - double starttime=0; - - log_debug(debug_level, "%d: get interval lock", pid); - - /* get the interval event from the hashtable */ - nthread_lock(&interval_mutex); - interval_iterator_t iter = interval_map.find(pid); - - /* case when interval is not found */ - if (iter == interval_map.end()) { - log_info(trace_debug_level, "invalid gid %d\n", gid); - nthread_unlock(&interval_mutex); - return -1; - } - - /* make sure the stack has at least one element */ - if (iter->second.size() == 0) { - log_error(trace_debug_level, "Error empty interval stack for pid %d\n", pid); - nthread_unlock(&interval_mutex); - return -1; - } - - /* The level is just the stack size */ - level = iter->second.size() - 1; - - starttime = iter->second.top(); - iter->second.pop(); - duration = endtime - starttime; - - log_debug(debug_level, "%d: release interval mutex", pid); - nthread_unlock(&interval_mutex); - - /* erase the entry if the interval size is zero */ - //if (iter->second.size() == 0) { - // interval_map.erase(iter); - //} - - - /* output the interval event */ - rc = tracefile->output_interval_event(eventID, pid, - level, data, duration); - } - - log_debug(debug_level, "%d: trace_end_interval(gid=%d) end", - pid,gid); - - return rc; -} - -/** - * @brief Generate an interval start event. - * - * The \ref trace_start_interval "trace_start_interval()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * We use the generic pablo trace and encode the additional information - * we want in the data field. The new data field will be, "interval:$name:duration". - * - * Pablo has its own interval records, but they are inadequate because it is - * difficult to measure concurrent intervals (e.g., in threads). - * A better (and more efficient) way to do this would be to create our own - * Pablo record type, but this is a quick "hack" to address our needs. - * - * @param interval_id @input_type The ID of the interval (unique for each interval) - * - * @return non-zero if successfull - * @return 0 if failure - */ -int trace_start_tput_interval( - const int gid, - const int pid) -{ - return trace_start_interval(gid, pid); -} - - -/** - * @brief Generate an end-interval event. - * - * The \ref trace_interval_event "trace_interval_event()" function generates an - * interval record that has information about the start, end, and duration of a - * particular code fragment. - * - * @param intervalID @input_type The interval ID (unique for each interval). - * @param eventID @input_type The ID of the trace. - * @param pid @input_type Process ID. - * @param level @input_type The depth of the interval (0=root, 1=sub-interval, 2=sub-sub,...). - * @param data @input_type User-defined data passed in a character - * string. - * - * @return non-zero if successfull - * @return 0 if failure - */ -int trace_end_tput_interval( - const int gid, - const int eventID, - const int pid, - const long num_processed, - const char *data) -{ - int level; - int rc = 0; - - if (tracefile && tracing_enabled(gid)) { - - double endtime = trios_get_time(); - double duration=0; - double starttime=0; - - /* get the interval event from the hashtable */ - nthread_lock(&interval_mutex); - interval_iterator_t iter = interval_map.find(pid); - - /* case when interval is not found */ - if (iter == interval_map.end()) { - fprintf(stderr, "Error: invalid gid\n"); - return -1; - } - - /* make sure the stack has at least one element */ - if (iter->second.size() == 0) { - fprintf(stderr, "Error empty interval stack for pid %d\n", pid); - return -1; - } - - starttime = iter->second.top(); - iter->second.pop(); - duration = endtime - starttime; - nthread_unlock(&interval_mutex); - - /* output the interval event */ - rc = tracefile->output_tput_event(eventID, pid, level, - data, duration, num_processed); - } - - return rc; -} - -int trace_get_count( - const int gid, - const int cid) -{ - int result = 0; - - if (tracing_enabled(gid)) { - - nthread_lock(&count_mutex); - count_iterator_t iter = count_map.find(cid); - - if (iter == count_map.end()) { - result = -1; - } - - else { - result = iter->second; - } - nthread_unlock(&count_mutex); - } - - return result; -} - -/** - * @brief Generate an increment-count event. - * - * The \ref trace_inc_count_event "trace_inc_count_event()" function - * increments a counter associated with the ID of counter and outputs - * a count event to the tracefile. - * - * @param id @input_type The ID of the counter. - * @param pid @input_type Process ID. - * @param data @input_type User-defined data in a character string. - * - * @returns 0 on sucess, -1 if the cid is invalid. - */ -int trace_inc_count( - const int gid, - const int cid, - const int pid, - const char *data) -{ - int rc = 0; - int val=0; - - - if ((tracefile != NULL) && tracing_enabled(gid)) { - - log_debug(trace_debug_level, "%d: before count_mutex lock",nthread_self()); - nthread_lock(&count_mutex); - log_debug(trace_debug_level, "%d: after count_mutex lock",nthread_self()); - - count_iterator_t iter = count_map.find(cid); - - if (iter == count_map.end()) { - trace_reset_count(gid, cid, pid, data); - iter = count_map.find(cid); - } - - val = ++(iter->second); - nthread_unlock(&count_mutex); - - tracefile->output_count_event(cid, pid, data, val); - } - - - return rc; -} - - -/** - * @brief Generate an decrement-count event. - * - * The \ref trace_inc_count_event "trace_inc_count_event()" function - * increments a counter associated with the ID of counter and outputs - * a count event to the tracefile. - * - * @param id @input_type The ID of the counter. - * @param pid @input_type Process ID. - * @param data @input_type User-defined data in a character string. - * - * @returns non-zero if successful. - * @returns 0 if failure. - */ -int trace_dec_count( - const int gid, - const int cid, - const int pid, - const char *data) -{ - int rc = 0; - int val; - - if (tracefile && tracing_enabled(gid)) { - - nthread_lock(&count_mutex); - count_iterator_t iter = count_map.find(cid); - - /* return error if a counter does not exist */ - if (iter == count_map.end()) { - log_warn(trace_debug_level, "%d: can't find cid=%d", cid); - rc = -1; - } - else { - val = --(iter->second); - rc = tracefile->output_count_event(cid, pid, data, val); - } - nthread_unlock(&count_mutex); - } - - return rc; -} - -/** - * @brief Generate a reset-count event. - * - * The \ref trace_reset_count "trace_reset_count()" function - * sets the value of a counter associated with the ID to 0 - * and outputs a count event to the tracefile. - * - * @param id @input_type The ID of the counter. - * @param pid @input_type Process ID. - * @param data @input_type User-defined data in a character string. - * - * @returns non-zero if successful. - * @returns 0 if failure. - */ -int trace_reset_count( - const int gid, - const int cid, - const int pid, - const char *data) -{ - return trace_set_count(gid, cid, pid, data, 0); -} - -/** - * @brief Generate an set-count event. - * - * The \ref trace_reset_count "trace_reset_count()" function - * sets the value of a counter associated with the ID - * and outputs a count event to the tracefile. - * - * @param id @input_type The ID of the counter. - * @param pid @input_type Process ID. - * @param data @input_type User-defined data in a character string. - * @param new_count @input_type New value for the counter. - * - * @returns non-zero if successful. - * @returns 0 if failure. - */ -int trace_set_count( - const int gid, - const int cid, - const int pid, - const char *data, - const int new_count) -{ - int rc = 0; - - if (tracefile && tracing_enabled(gid)) { - - nthread_lock(&count_mutex); - count_iterator_t iter = count_map.find(cid); - - /* if counter does not exist, create one */ - if (iter == count_map.end()) { - count_map.insert(pair< int, int>(cid, new_count)); - } - - else { - iter->second = new_count; - } - nthread_unlock(&count_mutex); - - rc = tracefile->output_count_event(cid, pid, data, new_count); - } - - return rc; -} - -struct CountWriter { - - const int pid; - const char *data; - - CountWriter(const int p, const char *d): pid(p), data(d) - { } - - void operator()(pair p) { - int val = p.second; - int cid = p.first; - if (tracefile) { - tracefile->output_count_event(cid, pid, data, val); - } - } -}; - -int trace_put_all_counts( - const int gid, - const int pid, - const char *data) -{ - int rc = 0; - - if (tracefile && tracing_enabled(gid)) { - CountWriter writer(pid, data); - - /* we need to be able to iterate through all entries in the hashtable */ - nthread_lock(&count_mutex); - for_each(count_map.begin(), count_map.end(), writer); - nthread_unlock(&count_mutex); - } - - return rc; -} - -#endif /* Trios_TRACIING_ENABLED */ diff --git a/packages/trios/libraries/support/trace/trace_noop.c b/packages/trios/libraries/support/trace/trace_noop.c deleted file mode 100644 index 71ae4fec1f00..000000000000 --- a/packages/trios/libraries/support/trace/trace_noop.c +++ /dev/null @@ -1,229 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file trace_noop.c - * - * @brief Noop implementation of trace API. - * - * @author Ron Oldfield (raoldfi@sandia.gov) - * @version $Revision: 406 $ - * @date $Date: 2005-10-07 15:08:29 -0600 (Fri, 07 Oct 2005) $ - * - */ - -#include "Trios_trace.h" - -/** - * A noop function that always returns 0. This function - * replaces all trace functions when TRACING_ENABLED is - * not set at compile time. - */ -int trace_noop(int a, ...) -{ - return 0; -} - -int trace_init(const char *trace_fname, const int trace_ftype) -{ - return 0; -} - -int trace_reset( - const char *trace_fname, - const int trace_ftype, - const char *enable) -{ - return 0; -} - -int trace_set_buffer_size(const unsigned long int bufsize) -{ - return 0; -} - -int trace_fini(void) -{ - return 0; -} - -int trace_register_group(const char *name, int *gid) -{ - return 0; -} - -int trace_register_subgroup(const char *name, int parentid, int *gid) -{ - return 0; -} - -int trace_get_gid(const char *name) -{ - return 0; -} - -const char *trace_get_gname(const int gid) -{ - return 0; -} - -int tracing_enabled(const int gid) -{ - return 0; -} - -int trace_enable(const char *name) -{ - return 0; -} - -int trace_enable_gid(const int gid) -{ - return 0; -} - -int trace_disable(const int gid) -{ - return 0; -} - -int trace_enable_all() -{ - return 0; -} - -int trace_disable_all() -{ - return 0; -} - -int trace_event( - const int gid, - const int eventID, - const int pid, - const char *data) -{ - return 0; -} - -int trace_start_interval( - const int gid, - const int pid) -{ - return 0; -} - -int trace_end_interval( - const int gid, - const int event_id, - const int pid, - const char *data) -{ - return 0; -} - -int trace_start_tput_interval(const int gid, const int pid) -{ - return 0; -} - -int trace_end_tput_interval( - const int gid, - const int event_id, - const int pid, - const long num_processed, - const char *data) -{ - return 0; -} - -int trace_get_count( - const int gid, - const int cid) -{ - return 0; -} - -int trace_inc_count( - const int gid, - const int cid, - const int pid, - const char *data) -{ - return 0; -} - - -int trace_dec_count( - const int gid, - const int cid, - const int pid, - const char *data) -{ - return 0; -} - -int trace_set_count( - const int gid, - const int cid, - const int pid, - const char *data, - const int newval) -{ - return 0; -} - -int trace_reset_count( - const int gid, - const int cid, - const int pid, - const char *data) -{ - return 0; -} - -int trace_put_all_counts( - const int gid, - const int pid, - const char *data) -{ - return 0; -} diff --git a/packages/trios/libraries/support/trace/trace_opts.ggo b/packages/trios/libraries/support/trace/trace_opts.ggo deleted file mode 100644 index c0f23cd10fa7..000000000000 --- a/packages/trios/libraries/support/trace/trace_opts.ggo +++ /dev/null @@ -1,45 +0,0 @@ -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - -option "trace-enable" - "Enable tracing on specified groups (comma-separated list)" string optional -option "trace-path" - "Path to the trace file" string optional -option "trace-ftype" -"Type of tracefile (0=binary,1=ASCII)" int default="0" optional - diff --git a/packages/trios/libraries/support/xdr/Readme b/packages/trios/libraries/support/xdr/Readme deleted file mode 100644 index ab7d918a29dd..000000000000 --- a/packages/trios/libraries/support/xdr/Readme +++ /dev/null @@ -1 +0,0 @@ -XDR is a required by nessie for \ No newline at end of file diff --git a/packages/trios/libraries/support/xdr/Trios_xdr.h b/packages/trios/libraries/support/xdr/Trios_xdr.h deleted file mode 100644 index 9bcdb7819275..000000000000 --- a/packages/trios/libraries/support/xdr/Trios_xdr.h +++ /dev/null @@ -1,102 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file Trios_xdr.h - * - * @brief Essential definitions and include files for code that uses XDR. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - */ - -#ifndef _TRIOS_XDR_H_ -#define _TRIOS_XDR_H_ - -#include "Trios_config.h" -#include -#include -#include - -#include -#include - -/* Some systems do not have xdr functions for uint16_t, uint32_t, uint64_t. - * We should have checked for these in the cmake config. To fix the problem, - * we redefine these function names. - */ -#ifdef HAVE_TRIOS_XDR_U_INT16_T -#define xdr_uint16_t xdr_u_int16_t -#endif -#ifdef HAVE_TRIOS_XDR_U_INT32_T -#define xdr_uint32_t xdr_u_int32_t -#endif -#ifdef HAVE_TRIOS_XDR_U_INT64_T -#define xdr_uint64_t xdr_u_int64_t -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - -/* On some systems (mac) xdr_sizeof is not in header file, - * but it is in the library. - */ -#ifndef HAVE_TRIOS_XDR_SIZEOF -extern unsigned long xdr_sizeof (xdrproc_t func, void *data); -#endif - - -#else /* K&R C */ -#endif - - - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/packages/trios/programs/CMakeLists.txt b/packages/trios/programs/CMakeLists.txt deleted file mode 100644 index 450e8f6d66be..000000000000 --- a/packages/trios/programs/CMakeLists.txt +++ /dev/null @@ -1,93 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -TRIBITS_SUBPACKAGE(programs) - -INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - - TRIBITS_ADD_EXECUTABLE( - kill-service - SOURCES nssi-kill.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX - ) - -# Gemini means Cray XE6. -# On Cray XE6, libtopomap requires the gpcd lib in addition to METIS and ParMETIS. -IF (TPL_ENABLE_Gemini AND - TPL_ENABLE_gpcd AND - TPL_ENABLE_METIS AND - TPL_ENABLE_ParMETIS AND - TPL_ENABLE_LibTopoMap) - TRIBITS_ADD_EXECUTABLE( - nssi-placement - SOURCES nssi-placement.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX - ) -ENDIF() - -# BGPDMA means SPI/DCMF which means BG/P. -# On BG/P, libtopomap only requires METIS and ParMETIS. -IF (TPL_ENABLE_BGPDMA AND - TPL_ENABLE_METIS AND - TPL_ENABLE_ParMETIS AND - TPL_ENABLE_LibTopoMap) - TRIBITS_ADD_EXECUTABLE( - nssi-placement - SOURCES nssi-placement.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX - ) -ENDIF() - - TRIBITS_ADD_EXECUTABLE( - ping-service - SOURCES nssi-ping.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX - ) - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/programs/cmake/Dependencies.cmake b/packages/trios/programs/cmake/Dependencies.cmake deleted file mode 100644 index f728f4865b9d..000000000000 --- a/packages/trios/programs/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS LibTopoMap ParMETIS METIS gpcd) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/programs/nssi-kill.cpp b/packages/trios/programs/nssi-kill.cpp deleted file mode 100644 index 39666a9f2684..000000000000 --- a/packages/trios/programs/nssi-kill.cpp +++ /dev/null @@ -1,200 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include "Trios_config.h" -#include "Trios_logger.h" -#include "Trios_nssi_client.h" - -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" - - -#include -#include -#include -#include - -//#include "nssi_types.h" - -#include -using namespace std; - -int admin_debug_level = LOG_UNDEFINED; - -/* ----------------- COMMAND-LINE OPTIONS --------------- */ - - -static void read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return; - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return; - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(admin_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); -} - -int kill_svc(nssi_service *svc, int sig, long timeout) -{ - int rc; - - - /* try to kill the remote service (TODO: need a timeout) */ - rc = nssi_kill(svc, sig, timeout); - if (rc != NSSI_OK) { - log_error(admin_debug_level, "could not kill svc: %s", - nssi_err_str(rc)); - return rc; - } - return rc; -} - -int -main (int argc, char *argv[]) -{ - int rc; - - // command-line arguments - int retries = 0; - int sig = 0; - int timeout = 1000; - log_level debug_level = LOG_ERROR; - string logfile(""); - - nssi_service svc; - char my_url[NSSI_URL_LEN]; - - std::string server_url(""); - char server_str[NSSI_URL_LEN]; - std::string contact_file(""); /* the file where the server's url should be written */ - - - try { - Teuchos::CommandLineProcessor parser; - - // init parser - parser.setDocString("Kill an NSSI server"); - - parser.setOption("verbose", (int *)(&debug_level), "Debug level."); - parser.setOption("logfile", &logfile, "Path to file for debug statements"); - parser.setOption("server-url", &server_url, "URL of NSSI service"); - parser.setOption("contact-file", &contact_file, "Where to read the server's URL"); - parser.setOption("timeout", &timeout, "Timout for contacting services (ms)"); - parser.setOption("retries", &retries, "Number of times to retry before exiting"); - parser.setOption("sig", &sig, "Signal to use for the kill command"); - - parser.recogniseAllOptions(); - parser.throwExceptions(); - - Teuchos::CommandLineProcessor::EParseCommandLineReturn - parseReturn= parser.parse( argc, argv ); - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - } - } - catch (...) { - exit(-1); - } - - /* initialize the logger */ - logger_init(debug_level, logfile.c_str()); - - if (server_url.c_str()[0]=='\0') { - sleep(1); - log_debug(debug_level, "reading URL from file"); - read_contact_info(contact_file.c_str(), server_str, NSSI_URL_LEN); - } else { - log_debug(debug_level, "using URL from command-line"); - strcpy(server_str, server_url.c_str()); - } - - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - nssi_get_url(NSSI_DEFAULT_TRANSPORT, my_url, NSSI_URL_LEN); - -// sleep(1); - log_info(debug_level, "\nTrying to get service at %s", server_str); - - rc=nssi_get_service(NSSI_DEFAULT_TRANSPORT, server_str, timeout, &svc); - if (rc != NSSI_OK) { - log_error(admin_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - return rc; - } - rc = kill_svc(&svc, sig, timeout); - if (rc == NSSI_ETIMEDOUT) { - fprintf(stderr, "Timed out trying to kill (%s)\n", - server_url.c_str()); - return rc; - } - else if (rc != NSSI_OK) { - log_error(admin_debug_level, "failed to kill service: %s", - nssi_err_str(rc)); - return rc; - } - - nssi_rpc_fini(NSSI_DEFAULT_TRANSPORT); - - return 0; -} diff --git a/packages/trios/programs/nssi-ping.cpp b/packages/trios/programs/nssi-ping.cpp deleted file mode 100644 index ea081b0d945a..000000000000 --- a/packages/trios/programs/nssi-ping.cpp +++ /dev/null @@ -1,184 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - - -#include "Trios_config.h" -#include "Trios_logger.h" -#include "Trios_nssi_client.h" - -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" - -#include - -#include -#include -#include - - -#include -#include - -int admin_debug_level = LOG_UNDEFINED; - -#define USE_CONTACT_INFO_FILE -#undef USE_CONTACT_INFO_FILE - - -static void read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return; - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return; - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(admin_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); -} - -int -main (int argc, char *argv[]) -{ - int rc = NSSI_OK; - int i; - nssi_service svc; - - // Command-line options - int retries = 1; - int sig = 0; - int timeout = 1000; - log_level debug_level = LOG_ERROR; - std::string logfile(""); - std::string server_url(""); - char server_str[NSSI_URL_LEN]; - char my_url[NSSI_URL_LEN]; - std::string contact_file(""); /* the file where the server's url should be written */ - - - try { - Teuchos::CommandLineProcessor parser; - - // init parser - parser.setDocString("Ping a remote Nessie server"); - - parser.setOption("verbose", (int *)(&debug_level), "Debug level."); - parser.setOption("logfile", &logfile, "Path to file for debug statements"); - parser.setOption("server-url", &server_url, "Portals network ID"); - parser.setOption("contact-file", &contact_file, "Where to read the server's URL"); - parser.setOption("timeout", &timeout, "Timout for contacting services (ms)"); - parser.setOption("retries", &retries, "Number of times to retry before exiting"); - parser.setOption("sig", &sig, "Signal to use for the kill command"); - - parser.recogniseAllOptions(); - parser.throwExceptions(); - - Teuchos::CommandLineProcessor::EParseCommandLineReturn - parseReturn= parser.parse( argc, argv ); - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - } - } - catch (...) { - exit(-1); - } - - /* initialize the logger */ - logger_init(debug_level, logfile.c_str()); - - if (server_url.c_str()[0]=='\0') { - sleep(1); - log_debug(debug_level, "reading URL from file"); - read_contact_info(contact_file.c_str(), server_str, NSSI_URL_LEN); - } else { - log_debug(debug_level, "using URL from command-line"); - strcpy(server_str, server_url.c_str()); - } - - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - nssi_get_url(NSSI_DEFAULT_TRANSPORT, my_url, NSSI_URL_LEN); - -// sleep(1); - log_info(debug_level, "\nPinging service at %s", server_str); - - for (i=0; i -#include - -#include -#include -#include -#include - -#include -using namespace std; - -int placement_debug_level = LOG_UNDEFINED; - -string strategy("ascending"); - -enum graph_connection_t { - GRAPH_COMPLETE=0, - GRAPH_CLIENT_COMPLETE, - GRAPH_SERVER_COMPLETE, - GRAPH_CLIENT_SERVER_ONLY -}; - - -/* ----------------- COMMAND-LINE OPTIONS --------------- */ - -#undef DEBUG_PLACEMENT - -void construct_graph( - int *rank_map, - int *nid_map, - int num_servers, - int num_clients, - int servers_per_node, - int clients_per_node, - int server_weight, - int client_weight, - int client_server_weight, - enum graph_connection_t connection, - int passthru) -{ - int npes, me, i, status; - int num_neighs; - std::vector neighbors; - std::vector weights; - - MPI_Comm_size(MPI_COMM_WORLD, &npes); - MPI_Comm_rank(MPI_COMM_WORLD, &me); - - const int len=128; - char my_hostname[len]; - int my_nid; - - gethostname(my_hostname, len); - my_nid = atoi(my_hostname+3); - -//#if defined(DEBUG_PLACEMENT) -// fprintf(stdout, "%i: hostname=%s, nid=%i\n", me, my_hostname, my_nid); -// sleep(2); -// MPI_Barrier(MPI_COMM_WORLD); -//#endif - - // Original rank_map - for (i = 0; i < npes; i++) { - rank_map[i] = i; - } - - MPI_Barrier(MPI_COMM_WORLD); - - if (me < num_servers) { - if ((connection == GRAPH_COMPLETE) || - (connection == GRAPH_SERVER_COMPLETE)) { - // construct a complete graph for servers-to-server communication - for (i=0;i 0) { - // clients also communicate with one server - int client_rank=(me-num_servers); // my rank within the clients - int bin_size=(num_clients/num_servers); // clients per server - int my_server=client_rank/bin_size; - neighbors.push_back(my_server); - weights.push_back(client_server_weight); - } - } - - MPI_Barrier(MPI_COMM_WORLD); - - num_neighs=neighbors.size(); - -#if defined(DEBUG_PLACEMENT) - for (i=0;i::iterator iter=neighbors.begin(); - for (;iter= num_servers) - cnf << nid_map[i] << "\t" << i << "\t" << rank_map[i] << std::endl; - } - - snf.close(); - cnf.close(); - } - - MPI_Finalize(); - - return 0; -} diff --git a/packages/trios/programs/nssi-trace-reset.cpp b/packages/trios/programs/nssi-trace-reset.cpp deleted file mode 100644 index c57c1b9c37f7..000000000000 --- a/packages/trios/programs/nssi-trace-reset.cpp +++ /dev/null @@ -1,153 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include "nssi_config.h" - -#include - -#include -#include -#include -using namespace std; - -#include "nssi_types.h" -#include "nssi_client.h" -#include "nssi_logger.h" -#include "nssi_CommandLineProcessor.hpp" -using namespace Nessie; - -log_level debug_level = LOG_UNDEFINED; - -enum TraceFtype {BINARY=0, ASCII=1}; - -int -main (int argc, char *argv[]) -{ - int rc; - nssi_service svc; - - // Command-line options - string addr; - int retries = 0; - int sig = 0; - int timeout = 1000; - log_level debug_level = LOG_ERROR; - std::string logfile(""); - std::string trace_enable(""); - std::string trace_path(""); - enum TraceFtype trace_ftype = BINARY; - - std::string server_url; - char my_url[NSSI_URL_LEN]; - - try { - Nessie::CommandLineProcessor parser; - - // init parser - parser.setDocString("Ping a remote Nessie server"); - - parser.setOption("verbose", (int *)(&debug_level), "Debug level."); - parser.setOption("logfile", &logfile, "Path to logfile"); - parser.setOption("server-url", &server_url, "Portals network ID"); - parser.setOption("timeout", &timeout, "Timout for contacting services (ms)"); - parser.setOption("retries", &retries, "Number of times to retry before exiting"); - parser.setOption("sig", &sig, "Signal to use for the kill command"); - parser.setOption("trace-enable", &trace_enable, - "Enable tracing on specified groups (comma-separated list)"); - parser.setOption("trace-path", &trace_path, "Path to trace file"); - - const int num_ftypes = 2; - const char *ftype_opt_names[] = {"binary","ascii"}; - const enum TraceFtype ftype_opt_values[] = {BINARY, ASCII}; - parser.setOption("trace-ftype", &trace_ftype, num_ftypes, - ftype_opt_values, ftype_opt_names, "File type of trace file"); - - parser.recogniseAllOptions(); - parser.throwExceptions(); - - Nessie::CommandLineProcessor::EParseCommandLineReturn - parseReturn= parser.parse( argc, argv ); - if( parseReturn == Nessie::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - if( parseReturn != Nessie::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - } - } - catch (...) { - exit(-1); - } - - - /* initialize the logger */ - logger_init(debug_level, logfile.c_str()); - - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - nssi_get_url(NSSI_DEFAULT_TRANSPORT, my_url, NSSI_URL_LEN); - - - rc=nssi_get_service(NSSI_DEFAULT_TRANSPORT, server_url.c_str(), timeout, &svc); - if (rc != NSSI_OK) { - log_error(debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - return rc; - } - - - /* reset the tracing on the remote service */ - rc = nssi_trace_reset(&svc, - trace_path.c_str(), - trace_ftype, - trace_enable.c_str(), - timeout); - if (rc != NSSI_OK) { - log_error(debug_level, "could not reset tracing: %s", - nssi_err_str(rc)); - return rc; - } - - nssi_rpc_fini(NSSI_DEFAULT_TRANSPORT); - - return 0; -} diff --git a/packages/trios/programs/nssi-url.cpp b/packages/trios/programs/nssi-url.cpp deleted file mode 100644 index 15e12fa27139..000000000000 --- a/packages/trios/programs/nssi-url.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include "nssi_config.h" - -#include -#include -#include -#include - -#include "nssi_client.h" - -/* ----------------- COMMAND-LINE OPTIONS --------------- */ - - -int -main (int argc, char *argv[]) -{ - char my_url[NSSI_URL_LEN]; - - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - nssi_get_url(NSSI_DEFAULT_TRANSPORT, my_url, NSSI_URL_LEN); - - printf("%s\n", my_url); - - nssi_rpc_fini(NSSI_DEFAULT_TRANSPORT); - - return 0; -} diff --git a/packages/trios/sampleScripts/config/chama/do-configure b/packages/trios/sampleScripts/config/chama/do-configure deleted file mode 100755 index cbf04b61f592..000000000000 --- a/packages/trios/sampleScripts/config/chama/do-configure +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Need Help? Go to http://trilinos.sandia.gov/Trilinos10CMakeQuickstart.txt - -# Notes on the cmake variables -# - The NSSI team uses PGI compilers by default. Mixing languages requires these flags - "-pgcpplibs -pgf90libs" -# - MPI details are in the toolchain file - -# Path to Trilinos source -TRILINOS_SRCDIR=$HOME/projects/trilinos/src/trilinos -TRIOS_SRCDIR=${TRILINOS_SRCDIR}/packages/trios - -# Load the appropriate bash environment -source ${TRIOS_SRCDIR}/sampleScripts/config/redsky/loadenv.sh - -rm -f CMakeCache.txt - - -EXTRA_ARGS=$@ - -INSTALL_PATH=$HOME/projects/trilinos/install/${TARGET_COMPILER} - -cmake \ -\ - -D CMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ -\ - -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \ -\ - -D TPL_ENABLE_MPI:BOOL=ON \ -\ - -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ - -D Trilinos_ENABLE_TESTS:BOOL=ON \ -\ - -D Trilinos_ENABLE_Trios:BOOL=ON \ - -D Trilinos_ENABLE_Triossupport:BOOL=ON \ - -D Trilinos_ENABLE_Triosnnti:BOOL=ON \ - -D Trilinos_ENABLE_Triosnssi:BOOL=ON \ - -D Trilinos_ENABLE_Triosprograms:BOOL=ON \ - -D Trilinos_ENABLE_Triostests:BOOL=ON \ - -D Trilinos_ENABLE_Triosexamples:BOOL=ON \ -\ - -D TPL_ENABLE_InfiniBand:BOOL=ON \ - -D InfiniBand_INCLUDE_DIRS:FILEPATH="/usr/include/infiniband" \ - -D InfiniBand_LIBRARY_DIRS:FILEPATH="/usr/lib64" \ -\ - $EXTRA_ARGS \ -\ - ${TRILINOS_SRCDIR} diff --git a/packages/trios/sampleScripts/config/chama/loadenv.sh b/packages/trios/sampleScripts/config/chama/loadenv.sh deleted file mode 100644 index 2004697ddbb9..000000000000 --- a/packages/trios/sampleScripts/config/chama/loadenv.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -module purge - -case $TARGET_COMPILER in - gnu) - module load gnu/4.7.2 - module load openmpi-gnu/1.6 - ;; - pgi) - module load pgi/12.5 - module load openmpi-pgi/1.6 - ;; - intel | *) - module load intel/12.1 - module load openmpi-intel/1.6 - ;; -esac - -module list diff --git a/packages/trios/sampleScripts/config/chama/toolchain-chama-gnu.cmake b/packages/trios/sampleScripts/config/chama/toolchain-chama-gnu.cmake deleted file mode 100644 index 90de86764896..000000000000 --- a/packages/trios/sampleScripts/config/chama/toolchain-chama-gnu.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_EXEC mpiexec) diff --git a/packages/trios/sampleScripts/config/chama/toolchain-chama-intel.cmake b/packages/trios/sampleScripts/config/chama/toolchain-chama-intel.cmake deleted file mode 100644 index 90de86764896..000000000000 --- a/packages/trios/sampleScripts/config/chama/toolchain-chama-intel.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_EXEC mpiexec) diff --git a/packages/trios/sampleScripts/config/chama/toolchain-chama-pgi.cmake b/packages/trios/sampleScripts/config/chama/toolchain-chama-pgi.cmake deleted file mode 100644 index 90de86764896..000000000000 --- a/packages/trios/sampleScripts/config/chama/toolchain-chama-pgi.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_EXEC mpiexec) diff --git a/packages/trios/sampleScripts/config/cielo/do-configure b/packages/trios/sampleScripts/config/cielo/do-configure deleted file mode 100755 index fae6bdadc668..000000000000 --- a/packages/trios/sampleScripts/config/cielo/do-configure +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -# Need Help? Go to http://trilinos.sandia.gov/Trilinos10CMakeQuickstart.txt - -# Notes on the cmake variables -# - the -lz is for the compression library (commsplitter uses crc) -# - MPI details are in the toolchain file - -# Path to Trilinos source -TRILINOS_SRCDIR=${HOME}/research/workspace/Trilinos -TRIOS_SRCDIR=${TRILINOS_SRCDIR}/packages/trios - -# Path to MPI (these environment variables should be set by modules) -MPI_HOME=${MPICH_DIR} - -rm -f CMakeCache.txt - -export TARGET_COMPILER=pgi - -# Load the appropriate bash environment -source ${TRIOS_SRCDIR}/sampleScripts/config/cielo/loadenv.sh - -TOOLCHAIN=${TRIOS_SRCDIR}/sampleScripts/config/cielo/toolchain-cnl-${TARGET_COMPILER}.cmake - -INSTALL_PATH=/home/raoldfi/research/support/cnl/lib/trilinos - -EXTRA_ARGS=$@ - -cmake \ - -D CMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ - -D CMAKE_C_COMPILER=cc \ - -D CMAKE_CXX_COMPILER=CC \ - -D CMAKE_Fortran_COMPILER=ftn \ -\ - -D BUILD_SHARED_LIBS:BOOL=OFF \ - -D TPL_FIND_SHARED_LIBS:BOOL=OFF \ - -D Trilinos_LINK_SEARCH_START_STATIC:BOOL=ON \ -\ - -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ - -D Trilinos_ENABLE_TESTS:BOOL=ON \ - -D Trilinos_EXTRA_LINK_FLAGS:STRING="-lz" \ -\ - -D Trilinos_ENABLE_Trios:BOOL=ON \ - -D Trilinos_ENABLE_Triossupport:BOOL=ON \ - -D Trilinos_ENABLE_Triosnnti:BOOL=ON \ - -D Trilinos_ENABLE_Trioscommsplitter:BOOL=ON \ - -D Trilinos_ENABLE_Triosnssi:BOOL=ON \ - -D Trilinos_ENABLE_Triosprograms:BOOL=ON \ - -D Trilinos_ENABLE_Triostests:BOOL=ON \ - -D Trilinos_ENABLE_Triosexamples:BOOL=ON \ -\ - -D TPL_ENABLE_MPI:BOOL=ON \ - -D MPI_LIBRARY=mpich \ - -D MPI_EXEC=aprun \ - -D MPI_EXEC_NUMPROCS_FLAG:STRING="-n" \ - -D MPI_USE_COMPILER_WRAPPERS:BOOL=OFF \ -\ - -D TPL_ENABLE_BLAS:STRING=ON \ - -D BLAS_LIBRARY_DIRS:FILEPATH="/opt/pgi/default/linux86-64/default/lib" \ - -D BLAS_LIBRARY_NAMES:STRING="acml" \ - -D TPL_ENABLE_LAPACK:STRING=ON \ - -D LAPACK_LIBRARY_DIRS:FILEPATH="/opt/pgi/default/linux86-64/default/lib" \ - -D LAPACK_LIBRARY_NAMES:STRING="acml" \ - -D HAVE_TEUCHOS_BLASFLOAT:BOOL=TRUE \ - -D CXX_COMPLEX_BLAS_WORKS:BOOL=TRUE \ -\ - -D TPL_ENABLE_Gemini:BOOL=ON \ - -D Gemini_INCLUDE_DIRS:FILEPATH="/opt/cray/gni-headers/default/include" \ -\ - -D TPL_ENABLE_PAPI:BOOL=OFF \ - -D TPL_ENABLE_HPCToolkit:BOOL=ON \ - -D HPCToolkit_INCLUDE_DIRS:FILEPATH="${HOME}/research/support/include" \ - -D HPCToolkit_LIBRARY_DIRS:FILEPATH="${HOME}/research/support/lib/hpctoolkit" \ -\ - $EXTRA_ARGS \ -\ - ${TRILINOS_SRCDIR} diff --git a/packages/trios/sampleScripts/config/cielo/loadenv.sh b/packages/trios/sampleScripts/config/cielo/loadenv.sh deleted file mode 100644 index 2248ebb7ccd4..000000000000 --- a/packages/trios/sampleScripts/config/cielo/loadenv.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# This script gets sourced by do-configure. But it can also be soruced in your shell like this: -# bash> TARGET_COMPILER=pgi source loadenv.sh -# -# The steps taken below can be summarized as: -# - check that TARGET_COMPILER is set. -# - check that TARGET_COMPILER is set to a known value. -# - cleanup the module list by unloading all known PrgEnv modules. -# - load the correct PrgEnv for the TARGET_COMPILER -# - load compiler neutral modules -# - perform any last minute compiler specific module swaps - - -# Load the appropriate bash environment -. /opt/modules/default/init/bash - - -if [ -z $TARGET_COMPILER ]; then - echo "==============================" - echo "==============================" - echo "TARGET_COMPILER is not defined in your environment. Please use one of the known values - pgi, gnu, cray or intel. Aborting." - echo "==============================" - echo "==============================" - return -fi - -case ${TARGET_COMPILER} in - pgi|gnu|cray|intel) - # TARGET_COMPILER is recognized. Carry on. - ;; - *) - echo "==============================" - echo "==============================" - echo "'$TARGET_COMPILER' is not a recognized value for TARGET_COMPILER. Please use one of the known values - pgi, gnu, cray or intel. Aborting." - echo "==============================" - echo "==============================" - return - ;; -esac - -# Unload any existing PrgEnv modules -module unload PrgEnv-pgi -module unload PrgEnv-gnu -module unload PrgEnv-cray -module unload PrgEnv-intel - -module load PrgEnv-${TARGET_COMPILER} -module load craype-hugepages2m -module load craype-hugepages2M - -module unload papi -module unload xt-papi -module unload xt-libsci - -if [ -z $VALGRIND_BUILD ]; then - # These specific module versions are required (defaults don't work) - echo "Loading fftw" - module load fftw - module load papi -else - echo "Unloading libsci and fftw" - # Unload any problematic modules - module unload xt-libsci - module unload fftw -fi - -# If you need to something special for your compiler. Do it here. -case ${TARGET_COMPILER} in - pgi) - ;; - gnu) - ;; - cray) - ;; - intel) - ;; -esac - - -module list diff --git a/packages/trios/sampleScripts/config/cielo/toolchain-cnl-gnu.cmake b/packages/trios/sampleScripts/config/cielo/toolchain-cnl-gnu.cmake deleted file mode 100644 index b9fbcbcec96c..000000000000 --- a/packages/trios/sampleScripts/config/cielo/toolchain-cnl-gnu.cmake +++ /dev/null @@ -1,16 +0,0 @@ -# Cross-compiler setup -set(CMAKE_C_COMPILER cc) -set(CMAKE_CXX_COMPILER CC) -set(CMAKE_Fortran_COMPILER ftn) - -# Gemini configuration -set(GEMINI_INCLUDE_DIRS /opt/cray/gni-headers/default/include) - -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_LIBRARY mpich) -set(MPI_EXEC aprun) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_USE_COMPILER_WRAPPERS OFF) -set(MPI_INCLUDE_PATH /opt/cray/mpt/default/xt/gemini/mpich2-gnu/include) -set(TPL_MPI_INCLUDE_DIRS /opt/cray/mpt/default/xt/gemini/mpich2-gnu/include) diff --git a/packages/trios/sampleScripts/config/cielo/toolchain-cnl-pgi.cmake b/packages/trios/sampleScripts/config/cielo/toolchain-cnl-pgi.cmake deleted file mode 100644 index 27e5ac740261..000000000000 --- a/packages/trios/sampleScripts/config/cielo/toolchain-cnl-pgi.cmake +++ /dev/null @@ -1,16 +0,0 @@ -# Cross-compiler setup -set(CMAKE_C_COMPILER cc) -set(CMAKE_CXX_COMPILER CC) -set(CMAKE_Fortran_COMPILER ftn) - -# Gemini configuration -set(GEMINI_INCLUDE_DIRS /opt/cray/gni-headers/default/include) - -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_LIBRARY mpich) -set(MPI_EXEC aprun) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_USE_COMPILER_WRAPPERS OFF) -set(MPI_INCLUDE_PATH /opt/cray/mpt/default/xt/gemini/mpich2-pgi/include) -set(TPL_MPI_INCLUDE_DIRS /opt/cray/mpt/default/xt/gemini/mpich2-pgi/include) diff --git a/packages/trios/sampleScripts/config/macos/checkin-test-trios.sh b/packages/trios/sampleScripts/config/macos/checkin-test-trios.sh deleted file mode 100755 index b56e62df04b0..000000000000 --- a/packages/trios/sampleScripts/config/macos/checkin-test-trios.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# -# Checkin-test script for Trios in the preCopyrightTrilinos repository. -# -# For macos with OpenMPI installed in /opt/local. -# -# We need to make sure Trios configures and builds with and without -# an RDMA transport library. On some machines, this won't be -# available. -# -# The Trios checkin test only evaluates the MPI_DEBUG build. Do we -# need to have a SERIAL_RELEASE build? -# - -# Use this to pass in addtitional args (e.g., --push) -EXTRA_ARGS=$@ - -# Set this point to your source -TRILINOS_SRC_DIR=${HOME}/research/workspace/Trilinos - -echo " --D BUILD_SHARED_LIBS:BOOL=ON --D TPL_ENABLE_Portals:BOOL=ON --D Portals_INCLUDE_DIRS:PATH=${HOME}/research/support/lib/portals/include --D Portals_LIBRARY_DIRS:PATH=${HOME}/research/support/lib/portals/lib -" > COMMON.config - -echo " --D TPL_ENABLE_MPI:BOOL=ON --D MPI_BASE_DIR:PATH=/opt/local/lib/openmpi -" > MPI_DEBUG.config - -# --without-serial-release \ - -# Test the MPI_DEBUG. This script will do pull, configure, build, test. -${TRILINOS_SRC_DIR}/checkin-test.py \ - --no-eg-git-version-check \ - --trilinos-src-dir=${TRILINOS_SRC_DIR} \ - --enable-all-packages=off \ - --no-enable-fwd-packages \ - --enable-packages=Trios,Triossupport,Triosnnti,Triosnssi,Triosprograms,Triostests,Triosexamples \ - --extra-build=MPI_DEBUG \ - --pull \ - --configure \ - --build \ - --test \ - --send-email-to='' \ - $EXTRA_ARGS - -# If we have stuff in preCopyrightTrilinos, this is how we enable it -# --extra-repos=preCopyrightTrilinos \ diff --git a/packages/trios/sampleScripts/config/macos/do-configure b/packages/trios/sampleScripts/config/macos/do-configure deleted file mode 100755 index defcef402828..000000000000 --- a/packages/trios/sampleScripts/config/macos/do-configure +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -EXTRA_ARGS=$@ - - - -rm -f CMakeCache.txt - -cmake \ -\ - -D CMAKE_BUILD_TYPE:STRING=DEBUG \ - -D CMAKE_CXX_FLAGS:STRING="-DMPI_INCLUDED" \ - -D CMAKE_C_FLAGS:STRING="-DMPI_INCLUDED" \ - -D CMAKE_VERBOSE_MAKEFILE:BOOL=TRUE \ -\ - -D BUILD_SHARED_LIBS:BOOL=ON \ -\ - -D TPL_ENABLE_MPI:BOOL=ON \ - -D MPI_USE_COMPILER_WRAPPERS:BOOL=ON \ - -D MPI_BASE_DIR:PATH=/opt/local/lib/openmpi \ -\ - -D HAVE_GCC_ABI_DEMANGLE:BOOL=ON \ - -D DART_TESTING_TIMEOUT:STRING=60 \ -\ - -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ - -D Trilinos_ENABLE_TESTS:BOOL=ON \ -\ - -D Trilinos_ENABLE_Trios:BOOL=ON \ - -D Trilinos_ENABLE_Triossupport:BOOL=ON \ - -D Trilinos_ENABLE_Triosnnti:BOOL=ON \ - -D Trilinos_ENABLE_Triosnssi:BOOL=ON \ - -D Trilinos_ENABLE_Triosprograms:BOOL=ON \ - -D Trilinos_ENABLE_Triostests:BOOL=ON \ - -D Trilinos_ENABLE_Triosexamples:BOOL=ON \ -\ - -D TPL_ENABLE_Portals:BOOL=ON \ - -D Portals_INCLUDE_DIRS:PATH=${HOME}/research/support/macos/portals/include \ - -D Portals_LIBRARY_DIRS:PATH=${HOME}/research/support/macos/portals/lib \ -\ - $EXTRA_ARGS \ -\ - ${HOME}/research/workspace/Trilinos - - diff --git a/packages/trios/sampleScripts/config/redsky/do-configure b/packages/trios/sampleScripts/config/redsky/do-configure deleted file mode 100755 index cbf04b61f592..000000000000 --- a/packages/trios/sampleScripts/config/redsky/do-configure +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Need Help? Go to http://trilinos.sandia.gov/Trilinos10CMakeQuickstart.txt - -# Notes on the cmake variables -# - The NSSI team uses PGI compilers by default. Mixing languages requires these flags - "-pgcpplibs -pgf90libs" -# - MPI details are in the toolchain file - -# Path to Trilinos source -TRILINOS_SRCDIR=$HOME/projects/trilinos/src/trilinos -TRIOS_SRCDIR=${TRILINOS_SRCDIR}/packages/trios - -# Load the appropriate bash environment -source ${TRIOS_SRCDIR}/sampleScripts/config/redsky/loadenv.sh - -rm -f CMakeCache.txt - - -EXTRA_ARGS=$@ - -INSTALL_PATH=$HOME/projects/trilinos/install/${TARGET_COMPILER} - -cmake \ -\ - -D CMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ -\ - -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \ -\ - -D TPL_ENABLE_MPI:BOOL=ON \ -\ - -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ - -D Trilinos_ENABLE_TESTS:BOOL=ON \ -\ - -D Trilinos_ENABLE_Trios:BOOL=ON \ - -D Trilinos_ENABLE_Triossupport:BOOL=ON \ - -D Trilinos_ENABLE_Triosnnti:BOOL=ON \ - -D Trilinos_ENABLE_Triosnssi:BOOL=ON \ - -D Trilinos_ENABLE_Triosprograms:BOOL=ON \ - -D Trilinos_ENABLE_Triostests:BOOL=ON \ - -D Trilinos_ENABLE_Triosexamples:BOOL=ON \ -\ - -D TPL_ENABLE_InfiniBand:BOOL=ON \ - -D InfiniBand_INCLUDE_DIRS:FILEPATH="/usr/include/infiniband" \ - -D InfiniBand_LIBRARY_DIRS:FILEPATH="/usr/lib64" \ -\ - $EXTRA_ARGS \ -\ - ${TRILINOS_SRCDIR} diff --git a/packages/trios/sampleScripts/config/redsky/loadenv.sh b/packages/trios/sampleScripts/config/redsky/loadenv.sh deleted file mode 100644 index 9750df7ce777..000000000000 --- a/packages/trios/sampleScripts/config/redsky/loadenv.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -module purge - -case $TARGET_COMPILER in - gnu) - module load gnu/4.7.1 - module load openmpi-gnu/1.6 - ;; - pgi) - module load compilers/pgi-12.8 - module load mpi/openmpi-1.4.3_oobpr_pgi-12.8-0 - ;; - intel | *) - module load compilers/intel-11.1-f064-c064 - module load mpi/openmpi-1.4.3_oobpr_intel-11.1-f064-c064 - ;; -esac - -module list - - diff --git a/packages/trios/sampleScripts/config/redsky/toolchain-redsky-gnu.cmake b/packages/trios/sampleScripts/config/redsky/toolchain-redsky-gnu.cmake deleted file mode 100644 index 90de86764896..000000000000 --- a/packages/trios/sampleScripts/config/redsky/toolchain-redsky-gnu.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_EXEC mpiexec) diff --git a/packages/trios/sampleScripts/config/redsky/toolchain-redsky-intel.cmake b/packages/trios/sampleScripts/config/redsky/toolchain-redsky-intel.cmake deleted file mode 100644 index 90de86764896..000000000000 --- a/packages/trios/sampleScripts/config/redsky/toolchain-redsky-intel.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_EXEC mpiexec) diff --git a/packages/trios/sampleScripts/config/redsky/toolchain-redsky-pgi.cmake b/packages/trios/sampleScripts/config/redsky/toolchain-redsky-pgi.cmake deleted file mode 100644 index 90de86764896..000000000000 --- a/packages/trios/sampleScripts/config/redsky/toolchain-redsky-pgi.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_EXEC_NUMPROCS_FLAG -n) -set(MPI_EXEC mpiexec) diff --git a/packages/trios/sampleScripts/config/redstorm/do-configure b/packages/trios/sampleScripts/config/redstorm/do-configure deleted file mode 100755 index ec9494a7d30a..000000000000 --- a/packages/trios/sampleScripts/config/redstorm/do-configure +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Need Help? Go to http://trilinos.sandia.gov/Trilinos10CMakeQuickstart.txt - -# Notes on the cmake variables -# - MPI details are in the toolchain file - -# Path to Trilinos source -TRILINOS_SRCDIR=${HOME}/research/workspace/Trilinos -TRIOS_SRCDIR=${TRILINOS_SRCDIR}/packages/trios - -# Load the appropriate bash environment -source ${TRIOS_SRCDIR}/sampleScripts/config/redstorm/loadenv.sh - -rm -f CMakeCache.txt - - -cmake \ - -D CMAKE_INSTALL_PREFIX=${INSTALL_PATH} \ - -D BUILD_SHARED_LIBS:BOOL=OFF \ - -D TPL_FIND_SHARED_LIBS:BOOL=OFF \ - -D Trilinos_LINK_SEARCH_START_STATIC:BOOL=ON \ -\ - -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ - -D Trilinos_ENABLE_TESTS:BOOL=ON \ -\ - -D CMAKE_CXX_FLAGS:STRING="-DMPICH_IGNORE_CXX_SEEK" \ - -D CMAKE_CXX_FLAGS_DEBUG:STRING="-DMPICH_IGNORE_CXX_SEEK" \ - -D CMAKE_CXX_FLAGS_RELEASE:STRING="-DMPICH_IGNORE_CXX_SEEK" \ -\ - -D Trilinos_ENABLE_Trios:BOOL=ON \ - -D Trilinos_ENABLE_Triossupport:BOOL=ON \ - -D Trilinos_ENABLE_Triosnnti:BOOL=ON \ - -D Trilinos_ENABLE_Triosnssi:BOOL=ON \ - -D Trilinos_ENABLE_Triosprograms:BOOL=ON \ - -D Trilinos_ENABLE_Triostests:BOOL=ON \ - -D Trilinos_ENABLE_Triosexamples:BOOL=ON \ -\ - -D TPL_ENABLE_MPI:BOOL=ON \ -\ - -D TPL_ENABLE_Pthread:STRING=OFF \ -\ - -D TPL_ENABLE_BLAS:STRING=ON \ - -D BLAS_LIBRARY_DIRS:FILEPATH="/opt/pgi/default/linux86-64/default/lib" \ - -D BLAS_LIBRARY_NAMES:STRING="acml" \ - -D TPL_ENABLE_LAPACK:STRING=ON \ - -D LAPACK_LIBRARY_DIRS:FILEPATH="/opt/pgi/default/linux86-64/default/lib" \ - -D LAPACK_LIBRARY_NAMES:STRING="acml" \ - -D HAVE_TEUCHOS_BLASFLOAT:BOOL=TRUE \ - -D CXX_COMPLEX_BLAS_WORKS:BOOL=TRUE \ -\ - -D TPL_ENABLE_CrayPortals:BOOL=ON \ - -D CrayPortals_INCLUDE_DIRS:FILEPATH="/opt/xt-pe/default/include" \ - "$@" \ - ${TRILINOS_SRCDIR} - diff --git a/packages/trios/sampleScripts/config/redstorm/loadenv.sh b/packages/trios/sampleScripts/config/redstorm/loadenv.sh deleted file mode 100644 index d4fb2a4615be..000000000000 --- a/packages/trios/sampleScripts/config/redstorm/loadenv.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Load the appropriate bash environment -. /opt/modules/default/init/bash - -# Unload GNU modules for catamount -module load PrgEnv-pgi -module load xt-papi - -# These two specific modules are required (defaults don't work) -module load fftw diff --git a/packages/trios/sampleScripts/config/redstorm/toolchain-catamount-pgi.cmake b/packages/trios/sampleScripts/config/redstorm/toolchain-catamount-pgi.cmake deleted file mode 100644 index 433e9c8b6544..000000000000 --- a/packages/trios/sampleScripts/config/redstorm/toolchain-catamount-pgi.cmake +++ /dev/null @@ -1,21 +0,0 @@ -# Cross-compiler setup -set(CMAKE_SYSTEM_NAME Catamount) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_C_COMPILER cc) -set(CMAKE_C_FLAGS "--target=catamount") -set(CMAKE_CXX_COMPILER CC) -set(CMAKE_CXX_FLAGS "--target=catamount") -set(CMAKE_Fortran_COMPILER ftn) -set(CMAKE_Fortran_FLAGS "--target=catamount") - -# Gemini configuration -set(PORTALS_INCLUDE_DIRS /opt/xt-pe/default/include) - -# MPI Configuration -set(TPL_ENABLE_MPI ON) -set(MPI_LIBRARY mpich) -set(MPI_EXEC yod) -set(MPI_EXEC_NUMPROCS_FLAG -sz) -set(MPI_USE_COMPILER_WRAPPERS OFF) -set(MPI_INCLUDE_PATH /opt/xt-mpt/default/mpich2-64/T/include) -set(TPL_MPI_INCLUDE_DIRS /opt/xt-mpt/default/mpich2-64/T/include) diff --git a/packages/trios/sampleScripts/config/sith/do-configure b/packages/trios/sampleScripts/config/sith/do-configure deleted file mode 100755 index 8d90865cc744..000000000000 --- a/packages/trios/sampleScripts/config/sith/do-configure +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -# Need Help? Go to http://trilinos.sandia.gov/Trilinos10CMakeQuickstart.txt - -# Notes on the cmake variables -# - The NSSI team uses PGI compilers by default. Mixing languages requires these flags - "-pgcpplibs -pgf90libs" -# - MPI details are in the toolchain file - -# Path to Trilinos source -TRILINOS_SRCDIR=$HOME/projects/trilinos/src/trilinos -TRIOS_SRCDIR=${TRILINOS_SRCDIR}/packages/trios - -# Load the appropriate bash environment -source ${TRIOS_SRCDIR}/sampleScripts/config/sith/loadenv.sh - -rm -f CMakeCache.txt - -TARGET_COMPILER=pgi -TOOLCHAIN=${TRIOS_SRCDIR}/sampleScripts/config/sith/toolchain-sith-${TARGET_COMPILER}.cmake - -EXTRA_ARGS=$@ - -INSTALL_PATH=$HOME/projects/trilinos/install/${TARGET_COMPILER} - -cmake \ - -D CMAKE_INSTALL_PREFIX=${INSTALL_PATH} \ - -D CMAKE_TOOLCHAIN_FILE=${TOOLCHAIN} \ -\ - -D CMAKE_BUILD_TYPE:STRING=DEBUG \ -\ - -D CMAKE_CXX_FLAGS:STRING="-pgf90libs" \ - -D CMAKE_C_FLAGS:STRING="-pgcpplibs -pgf90libs" \ -\ - -D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \ -\ - -D TPL_ENABLE_Pthread:BOOL=ON \ - -D TPL_ENABLE_MPI:BOOL=ON \ -\ - -D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ - -D Trilinos_ENABLE_TESTS:BOOL=ON \ -\ - -D Trilinos_ENABLE_Trios:BOOL=ON \ -\ - -D TPL_ENABLE_InfiniBand:BOOL=ON \ - -D InfiniBand_INCLUDE_DIRS:FILEPATH="/usr/include/infiniband" \ - -D InfiniBand_LIBRARY_DIRS:FILEPATH="/usr/lib64" \ -\ - -D TPL_ENABLE_BLAS:STRING=ON \ - -D BLAS_LIBRARY_DIRS:FILEPATH="/sw/sith/blas/ref/centos5.5_pgi10.9/lib" \ - -D BLAS_LIBRARY_NAMES:STRING="blas" \ - -D TPL_ENABLE_LAPACK:STRING=ON \ - -D LAPACK_LIBRARY_DIRS:FILEPATH="/sw/sith/lapack/3.1.1/centos5.5_pgi10.9_shared/lib" \ - -D LAPACK_LIBRARY_NAMES:STRING="lapack" \ -\ - $EXTRA_ARGS \ -\ - ${TRILINOS_SRCDIR} diff --git a/packages/trios/sampleScripts/config/sith/loadenv.sh b/packages/trios/sampleScripts/config/sith/loadenv.sh deleted file mode 100644 index a66ebf547e98..000000000000 --- a/packages/trios/sampleScripts/config/sith/loadenv.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -module swap ompi/1.4.2-pgi10.9 -module load blas -module load lapack - -module load cmake diff --git a/packages/trios/sampleScripts/config/sith/toolchain-sith-pgi.cmake b/packages/trios/sampleScripts/config/sith/toolchain-sith-pgi.cmake deleted file mode 100644 index 6cdbcdecf2d4..000000000000 --- a/packages/trios/sampleScripts/config/sith/toolchain-sith-pgi.cmake +++ /dev/null @@ -1,2 +0,0 @@ -# MPI Configuration -set(MPI_EXEC_NUMPROCS_FLAG -n) diff --git a/packages/trios/sampleScripts/config/x86_64/do-configure b/packages/trios/sampleScripts/config/x86_64/do-configure deleted file mode 100755 index aa8880fdd41f..000000000000 --- a/packages/trios/sampleScripts/config/x86_64/do-configure +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -EXTRA_ARGS=$@ - -#-D MPI_USE_COMPILER_WRAPPERS:BOOL=OFF \ - -rm -f CMakeCache.txt - -cmake \ -\ --D CMAKE_BUILD_TYPE:STRING=DEBUG \ --D CMAKE_CXX_FLAGS:STRING="-DMPI_INCLUDED" \ --D CMAKE_C_FLAGS:STRING="-DMPI_INCLUDED" \ --D CMAKE_VERBOSE_MAKEFILE:BOOL=TRUE \ -\ --D Trilinos_WARNINGS_AS_ERRORS_FLAGS:STRING="" \ --D TPL_ENABLE_MPI:BOOL=ON \ --D MPI_BASE_DIR:PATH=/home/raoldfi/research/support/ \ --D HAVE_GCC_ABI_DEMANGLE:BOOL=ON \ --D DART_TESTING_TIMEOUT:STRING=600 \ --D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \ --D Trilinos_ENABLE_TESTS:BOOL=ON \ -\ -\ --D Trilinos_ENABLE_Trios:BOOL=ON \ -\ --D TPL_ENABLE_Portals:BOOL=ON \ --D Portals_INCLUDE_DIRS:PATH=${HOME}/research/support/include \ --D Portals_LIBRARY_DIRS:PATH=${HOME}/research/support/lib \ -\ -$EXTRA_ARGS \ -\ -${HOME}/research/workspace/Trilinos - - diff --git a/packages/trios/services/CMakeLists.txt b/packages/trios/services/CMakeLists.txt deleted file mode 100644 index 3df6af12e844..000000000000 --- a/packages/trios/services/CMakeLists.txt +++ /dev/null @@ -1,48 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - -#IF (TRIOS_ENABLE_NETCDF_SERVICE AND -# TRIOS_ENABLE_PNETCDF_SERVICE) -# -# ADD_SUBDIRECTORY(netcdf) -# -#ENDIF() diff --git a/packages/trios/services/Readme b/packages/trios/services/Readme deleted file mode 100644 index 3577cd0ee7b8..000000000000 --- a/packages/trios/services/Readme +++ /dev/null @@ -1,23 +0,0 @@ -This directory contains source code to support Trios Data Services. - -Nessie (NEtwork Scalable Service Interface), also abbrivated as NSSI. - Nessie is a parallel client-server framework for data services. All code in - this directory is built using nessie. - -pvspy-service: - The pvspy-service is an in-transit service that provides staging and computation - for CTH. The client implements the pvspy API already used by CTH for in-situ - analysis. The server executes as a parallel ParaView application that uses the - co-processing features of ParaView to perform analysis on staged data. - -netcdf-service: - The netcdf service provides staging and aggregation for applications using the netcdf - or PnetCDF api. The server caches data and either aggregates data or simply - forwards netcdf calls from the client. - -sql-service: - The client provides an SQL-like interface. The server acts as a proxy, forwarding - SQL commands to any ODBC-capable server. This service is valuable on systems that - do not support traditional database clients. - - diff --git a/packages/trios/services/netcdf/CMakeLists.txt b/packages/trios/services/netcdf/CMakeLists.txt deleted file mode 100644 index f77a464e69c2..000000000000 --- a/packages/trios/services/netcdf/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - - -TRIBITS_SUBPACKAGE(netcdf-service) - -ADD_SUBDIRECTORY(src) - -TRIBITS_ADD_TEST_DIRECTORIES(tests) - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/services/netcdf/cmake/Dependencies.cmake b/packages/trios/services/netcdf/cmake/Dependencies.cmake deleted file mode 100644 index 6112c0e4eafd..000000000000 --- a/packages/trios/services/netcdf/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI Pnetcdf Netcdf) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/services/netcdf/src/CMakeLists.txt b/packages/trios/services/netcdf/src/CMakeLists.txt deleted file mode 100644 index 4b4dc3574033..000000000000 --- a/packages/trios/services/netcdf/src/CMakeLists.txt +++ /dev/null @@ -1,122 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - -INCLUDE(TriosProcessXDR) - - -INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/src) - - -SET(HEADERS "") -SET(NONINSTALLHEADERS "") -SET(SOURCES "") - -# Need to include the source dir so we can find aggregation.h -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - -# The rpcgen files should be treated as system files to avoid annoying warnings -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/netcdf_args.x) - - -APPEND_SET(NOINSTHEADERS - ${CMAKE_CURRENT_BINARY_DIR}/netcdf_args.h - aggregation.h - create_subchunks.h - io_timer.h - NcAttInfo.h - NcDimInfo.h - NcFileInfo.h - NcGroupInfo.h - NcVarInfo.h - netcdf_client_private.h - netcdf_debug.h -) - -APPEND_SET(LIB_SOURCES - ${CMAKE_CURRENT_BINARY_DIR}/netcdf_args.c - netcdf_debug.c - NcFileInfo.cpp - NcAttInfo.cpp - NcVarInfo.cpp - NcDimInfo.cpp - NcGroupInfo.cpp - netcdf_client.cpp - ncmpi_client.cpp - create_subchunks.cpp -) - -############# libtrios_netcdf_client ########################## - -TRIBITS_ADD_LIBRARY( - trios_netcdf_client - HEADERS ${HEADERS} - NOINSTALLHEADERS ${NOINSTALLHEADERS} - SOURCES ${LIB_SOURCES} - TESTONLY -) - -# trios_netcdf_client is created as a TESTONLY library which means it won't -# get installed. forced the install here. -INSTALL(FILES - ${CMAKE_CURRENT_BINARY_DIR}/libtrios_netcdf_client.a - DESTINATION "${${PROJECT_NAME}_INSTALL_LIB_DIR}" -) -GLOBAL_SET(${PACKAGE_NAME}_NETCDF_LIBRARIES "trios_netcdf_client") - -###################### PROGRAMS ################################################## - -INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) - -APPEND_SET(SVC_SOURCES - ${CMAKE_CURRENT_BINARY_DIR}/netcdf_args.c - netcdf_server.cpp - netcdf_debug.c - aggregation.cpp -) - -TRIBITS_ADD_EXECUTABLE( - netcdf-server - SOURCES ${SVC_SOURCES} - DEPLIBS ${Trios_LIBRARIES} - NOEXEPREFIX -) diff --git a/packages/trios/services/netcdf/src/NcAttInfo.cpp b/packages/trios/services/netcdf/src/NcAttInfo.cpp deleted file mode 100644 index ca2fbc7a3e18..000000000000 --- a/packages/trios/services/netcdf/src/NcAttInfo.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncAttr.cpp - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#include "Trios_config.h" -#ifdef HAVE_TRIOS_PNETCDF - -#include -#include -#include -#include "NcAttInfo.h" -#include "netcdf_args.h" -#include "netcdf_debug.h" - - -#if USE_NC_TYPE -NcAttInfo::NcAttInfo(const char *name, const nc_type xtype, const size_t len) : - _xtype(xtype), _name(name),_len(len) -#else -NcAttInfo::NcAttInfo(const char *name, const int xtype, const size_t len) : - _xtype(xtype), _name(name), _len(len) -#endif -{ -} - -NcAttInfo::NcAttInfo(const struct nc_att &att) : - _xtype(att.xtype), _name(att.name), _len(att.len) -{ - log_debug(netcdf_debug_level, "Created attribute (%s, type=%d, len=%d)", - _name.c_str(), (int)_xtype, (int)_len); -} - -NcAttInfo::~NcAttInfo() -{ -} - -int NcAttInfo::copyTo(struct nc_att &att) -{ - att.xtype = this->_xtype; - att.name = strdup(this->_name.c_str()); - att.len = this->_len; - - return NC_NOERR; -} - -#if USE_NC_TYPE -int NcAttInfo::inq_att(char *name, nc_type *xtypep, size_t *lenp) -#else -int NcAttInfo::inq_att(char *name, int *xtypep, size_t *lenp) -#endif -{ - if (name != NULL) { - strcpy(name, this->_name.c_str()); - } - - *xtypep = static_cast(this->_xtype); - *lenp = this->_len; - - return NC_NOERR; -} - - -int NcAttInfo::inq_attname(char *name) -{ - if (name != NULL) { - strcpy(name, this->_name.c_str()); - } - return NC_NOERR; -} - - -#if USE_NC_TYPE -int NcAttInfo::inq_atttype(nc_type *xtypep) -#else -int NcAttInfo::inq_atttype(int *xtypep) -#endif -{ - *xtypep = static_cast(this->_xtype); - return NC_NOERR; -} - - -int NcAttInfo::inq_attlen(size_t *attlenp) -{ - *attlenp = this->_len; - return NC_NOERR; -} - - - - -#endif // HAVE_TRIOS_PNETCDF diff --git a/packages/trios/services/netcdf/src/NcAttInfo.h b/packages/trios/services/netcdf/src/NcAttInfo.h deleted file mode 100644 index 04150fb87183..000000000000 --- a/packages/trios/services/netcdf/src/NcAttInfo.h +++ /dev/null @@ -1,127 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncAttr.h - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#ifndef NCATTR_H_ -#define NCATTR_H_ - -#include -using namespace std; - -#include "netcdf_args.h" - -#undef USE_NC_TYPE -#define USE_NC_TYPE 1 - - -/** Attributes may be associated with each netCDF variable to specify - * such properties as units, special values, maximum and minimum valid - * values, scaling factors, and offsets. Attributes for a netCDF dataset - * are defined when the dataset is first created, while the netCDF dataset - * is in define mode. Additional attributes may be added later by - * reentering define mode. A netCDF attribute has a netCDF variable - * to which it is assigned, a name, a type, a length, and a sequence - * of one or more values. An attribute is designated by its variable - * ID and name. When an attribute name is not known, it may be designated - * by its variable ID and number in order to determine its name, using - * the function nc_inq_attname. - * - * The attributes associated with a variable are typically defined - * immediately after the variable is created, while still in define mode. - * The data type, length, and value of an attribute may be changed even - * when in data mode, as long as the changed attribute requires no more - * space than the attribute as originally defined. - * - * It is also possible to have attributes that are not associated with any - * variable. These are called global attributes and are identified by using - * NC_GLOBAL as a variable pseudo-ID. Global attributes are usually related - * to the netCDF dataset as a whole and may be used for purposes such as - * providing a title or processing history for a netCDF dataset. - * - * The NcAttribute Class does not actually contain the attribute data, it - * just has the attribute metadata (id, name, type, len). Use the - * NcVariable::get_att methods to obtain the actual data. - */ - -class NcAttInfo { - -public: -#if USE_NC_TYPE - int inq_att(char *name, nc_type *xtypep, size_t *lenp); -#else - int inq_att(char *name, int *xtypep, size_t *lenp); -#endif - int inq_attname(char *name); -#if USE_NC_TYPE - int inq_atttype(nc_type *xtypep); -#else - int inq_atttype(int *xtypep); -#endif - int inq_attlen(size_t *attlenp); - - /** - * Convert to struct nc_att. - */ - int copyTo(struct nc_att &); - -#if USE_NC_TYPE - NcAttInfo(const char *name, const nc_type xtype, const size_t len); -#else - NcAttInfo(const char *name, const int xtype, const size_t len); -#endif - NcAttInfo(const nc_att &att); - virtual ~NcAttInfo(); - -private: - - int _xtype; - string _name; - size_t _len; -}; - -#endif /* NCATTR_H_ */ diff --git a/packages/trios/services/netcdf/src/NcDimInfo.cpp b/packages/trios/services/netcdf/src/NcDimInfo.cpp deleted file mode 100644 index fb0b7c08afce..000000000000 --- a/packages/trios/services/netcdf/src/NcDimInfo.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncDim.cpp - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#include "Trios_config.h" -#ifdef HAVE_TRIOS_PNETCDF - -#include -using namespace std; - -#include "Trios_nssi_types.h" - -#include -#include -#include "netcdf_args.h" -#include "NcDimInfo.h" - -NcDimInfo::NcDimInfo( - const int dimid, - const char *name, - const size_t len) : - _dimid(dimid), _name(name), _len(len) -{ -} - -NcDimInfo::NcDimInfo(const nc_dim &dim) : - _dimid(dim.dimid), _name(dim.name), _len(dim.len) -{ -} - -NcDimInfo::~NcDimInfo() { - -} - -/** - * Convert NcDimInfo into struct nc_dim. - */ -int NcDimInfo::copyTo(struct nc_dim &dim) -{ - dim.dimid = this->_dimid; - dim.name = strdup(this->_name.c_str()); - dim.len = this->_len; - - return NC_NOERR; -} - - -/** Get information about the dimension. */ -int NcDimInfo::inq_dim(char *name, size_t *lengthp) -{ - int rc = NC_NOERR; - inq_dimname(name); - inq_dimlen(lengthp); - return rc; -} - -/** Get the name of the dimension. */ -int NcDimInfo::inq_dimname(char *name) -{ - int rc = NC_NOERR; - strcpy(name, this->_name.c_str()); - return rc; -} - -/** Get the length of the dimension. */ -int NcDimInfo::inq_dimlen(size_t *lengthp) -{ - int rc = NC_NOERR; - *lengthp = this->_len; - return rc; -} - -/** Get the ID of the dimension. */ -int NcDimInfo::inq_dimid(int *dimid) -{ - int rc = NC_NOERR; - *dimid = this->_dimid; - return rc; -} - -/** Rename a dimension. */ -int NcDimInfo::rename_dim(const char *newname) -{ - int rc = NC_NOERR; - this->_name = string(newname); - return rc; -} - -#endif // HAVE_TRIOS_PNETCDF diff --git a/packages/trios/services/netcdf/src/NcDimInfo.h b/packages/trios/services/netcdf/src/NcDimInfo.h deleted file mode 100644 index 15fa312854b6..000000000000 --- a/packages/trios/services/netcdf/src/NcDimInfo.h +++ /dev/null @@ -1,102 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncDim.h - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#ifndef NCDIM_H_ -#define NCDIM_H_ - -#include - -class NcGroupInfo; - -/** - * Dimensions are defined while the dataset is in define mode. - * A netCDF dimension has a name and length. - */ -class NcDimInfo { - -public: - - /** Get information about the dimension. */ - int inq_dim(char *name, size_t *lengthp); - - /** Get the name of the dimension. */ - int inq_dimname(char *name); - - /** Get the length of the dimension. */ - int inq_dimlen(size_t *lengthp); - - /** Get the ID of the dimension. */ - int inq_dimid(int *dimid); - - /** Rename a dimension. */ - int rename_dim(const char *newname); - - /** Convert to struct nc_dim */ - int copyTo(struct nc_dim &); - - -public: - /** Create a new dimension. */ - NcDimInfo(const int dimid, const char *name, const size_t len); - - /** Creat a new dimension from struct nc_dim */ - NcDimInfo(const struct nc_dim &dim); - - virtual ~NcDimInfo(); - -private: - const int _dimid; - std::string _name; - size_t _len; - - friend class NcGroupInfo; - -}; - -#endif /* NCDIM_H_ */ diff --git a/packages/trios/services/netcdf/src/NcFileInfo.cpp b/packages/trios/services/netcdf/src/NcFileInfo.cpp deleted file mode 100644 index 7d067f195739..000000000000 --- a/packages/trios/services/netcdf/src/NcFileInfo.cpp +++ /dev/null @@ -1,86 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncDataset.cpp - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ -#include -#include "NcFileInfo.h" -#include "netcdf_args.h" - -NcFileInfo::NcFileInfo( - const char *path, - const int mode, - const size_t initialsz, - const size_t chunksize) -: _path(path), _mode(mode), _initialsz(initialsz), _chunksize(chunksize) -{ -} - -NcFileInfo::NcFileInfo(const NcFileInfo ©) -: _path(copy._path), _mode(copy._mode), _initialsz(copy._initialsz), _chunksize(copy._chunksize), _format(copy._format) -{ -} - - -NcFileInfo::~NcFileInfo() { -} - - -/** Set default creation format. */ -int NcFileInfo::set_format(const int format) -{ - int rc = NC_NOERR; - this->_format = format; - return rc; -} - -/** Return the format of the dataset. */ -int NcFileInfo::inq_format(int *formatp) -{ - int rc = NC_NOERR; - *formatp = this->_format; - return rc; -} diff --git a/packages/trios/services/netcdf/src/NcFileInfo.h b/packages/trios/services/netcdf/src/NcFileInfo.h deleted file mode 100644 index 8e3bc114c2c9..000000000000 --- a/packages/trios/services/netcdf/src/NcFileInfo.h +++ /dev/null @@ -1,92 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncDataset.h - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#ifndef NCFILEINFO_H_ -#define NCFILEINFO_H_ - - -#include -using namespace std; - -#include - - -class NcFileInfo { - -public: - - NcFileInfo (const char *path, const int cmode, - const size_t initialsz, const size_t chunksize); - - NcFileInfo(const NcFileInfo &); - - virtual ~NcFileInfo(); - - /** Set default creation format. */ - int set_format(const int format); - - /** Return the format of the dataset. */ - int inq_format(int *formatp); - - -private: - - const string _path; - const int _mode; - const size_t _initialsz; - const size_t _chunksize; - int _format; - - /** Constructor for netcdf Datasets. Only members of this class - * can call the constructor. - */ - NcFileInfo(); -}; - -#endif /* NCDATASET_H_ */ diff --git a/packages/trios/services/netcdf/src/NcGroupInfo.cpp b/packages/trios/services/netcdf/src/NcGroupInfo.cpp deleted file mode 100644 index 4280c3731d5a..000000000000 --- a/packages/trios/services/netcdf/src/NcGroupInfo.cpp +++ /dev/null @@ -1,889 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncGroup.cpp - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#include "Trios_config.h" -#ifdef HAVE_TRIOS_PNETCDF - -#include -#include -#include -#include - -#include -#include -#include -#include -using namespace std; - - -#include "netcdf_args.h" -#include "netcdf_debug.h" -#include "NcGroupInfo.h" -#include "NcVarInfo.h" -#include "NcDimInfo.h" -#include "NcFileInfo.h" -#include "NcAttInfo.h" - -#ifndef NC_ENOGRP -#define NC_ENOGRP (-125) /* no group found */ -#endif - -/** Global map of open groups (used by ncmpi_client and netcdf_client) */ -map group_map; - - -/** Create a new root group */ -NcGroupInfo::NcGroupInfo(const int ncid, const NcFileInfo &fileInfo) : - _ncid(ncid), _fileInfo(fileInfo), _parent(NULL), _name("/"), _unlimdimid(-1) -{ } - - -/** Create a new child group. */ -NcGroupInfo::NcGroupInfo(const int ncid, const char *name, NcGroupInfo &parent) : - _ncid(ncid), _fileInfo(parent._fileInfo), _parent(&parent), _name(name), _unlimdimid(-1) -{ -} - -NcGroupInfo::NcGroupInfo(const struct nc_group &group, NcGroupInfo &parent) : - _ncid(group.ncid), _fileInfo(parent._fileInfo), _parent(&parent) -{ - copyFrom(group); -} - - -/** Create a group class from the struct nc_group data structure. */ -NcGroupInfo::NcGroupInfo(const struct nc_group &group, const NcFileInfo &fileInfo) : - _ncid(group.ncid), _fileInfo(fileInfo), _parent(NULL) -{ - /* fill contents of this class with info from nc_group */ - copyFrom(group); -} - - -int NcGroupInfo::copyFrom(const struct nc_group &group) -{ - log_level debug_level = netcdf_debug_level; - - uint32_t i; - - this->_unlimdimid = group.unlimdimid; - - /* copy dimensions */ - log_debug(debug_level, "copy %d dims", group.dims.dims_len); - for (i=0; i_dims[dim->dimid] = new NcDimInfo(*dim); - } - - /* copy vars */ - log_debug(debug_level, "copy %d vars", group.vars.vars_len); - for (i=0; i_vars[var->varid] = this->_varsByName[var->name] = new NcVarInfo(*var); - } - - /* copy global attributes */ - log_debug(debug_level, "copy %d atts", group.atts.atts_len); - for (i=0; i_atts[att->name] = new NcAttInfo(*att); - } - - - /* copy subgroups */ - log_debug(debug_level, "copy %d groups", group.groups.groups_len); - for (i=0; i_children[subgrp->ncid] = new NcGroupInfo(*subgrp, *this); - } - - return 0; -} - - - - -struct delete_object -{ - template - void operator()(pair &p) { - dataT *data = p.second; - if (data) { - delete data; - data = 0; - } - } -}; - -NcGroupInfo::~NcGroupInfo() -{ - log_level debug_level = netcdf_debug_level; - - /* Delete variables */ - log_debug(debug_level, "Deleting vars (%d)", _vars.size()); - for_each(_vars.begin(), _vars.end(), delete_object()); - - /* Delete dimensions */ - log_debug(debug_level, "Deleting dims (%d)", _dims.size()); - for_each(_dims.begin(), _dims.end(), delete_object()); - - /* Delete attributes */ - log_debug(debug_level, "Deleting atts (%d)", _atts.size()); - for_each(_atts.begin(), _atts.end(), delete_object()); - - /* Delete subgroups */ - log_debug(debug_level, "Deleting children (%d)", _children.size()); - for_each(_children.begin(), _children.end(), delete_object()); -} - -/** - * Convert to a serializable struct nc_group. - */ -int NcGroupInfo::copyTo(struct nc_group &group) -{ - int ndims, nvars, natts, ngrps; - log_level debug_level = netcdf_debug_level; - - memset(&group, 0, sizeof(struct nc_group)); - - group.ncid = this->_ncid; - group.parent_ncid = (this->_parent)? this->_parent->_ncid: -1; - group.name = strdup(this->_name.c_str()); - - - /* copy dimensions */ - ndims = this->_dims.size(); - group.dims.dims_len = ndims; - log_debug(debug_level, "copy %d dims", ndims); - if (ndims) { - std::map::iterator dim_iter; - int i=0; - group.dims.dims_val = (struct nc_dim *)calloc(ndims, sizeof(struct nc_dim)); - for (dim_iter = this->_dims.begin(); dim_iter != this->_dims.end(); dim_iter++) { - dim_iter->second->copyTo(group.dims.dims_val[i++]); - } - } - - /* copy vars */ - nvars = this->_vars.size(); - group.vars.vars_len = nvars; - log_debug(debug_level, "copy %d vars", nvars); - if (nvars) { - std::map::iterator var_iter; - int i=0; - group.vars.vars_val = (struct nc_var *)calloc(nvars, sizeof(struct nc_var)); - for (var_iter = this->_vars.begin(); var_iter != this->_vars.end(); var_iter++) { - var_iter->second->copyTo(group.vars.vars_val[i++]); - } - } - - /* copy global attributes */ - natts = this->_atts.size(); - group.atts.atts_len = natts; - log_debug(debug_level, "copy %d atts", natts); - if (natts) { - std::map::iterator att_iter; - int i=0; - group.atts.atts_val = (struct nc_att *)calloc(natts, sizeof(struct nc_att)); - for (att_iter = this->_atts.begin(); att_iter != this->_atts.end(); att_iter++) { - att_iter->second->copyTo(group.atts.atts_val[i++]); - } - } - - /* copy subgroups */ - ngrps = this->_children.size(); - group.groups.groups_len = ngrps; - log_debug(debug_level, "copy %d subgroups", ngrps); - if (ngrps) { - std::map::iterator grp_iter; - int i=0; - group.groups.groups_val = (struct nc_group *)calloc(ngrps, sizeof(struct nc_group)); - for (grp_iter = this->_children.begin(); grp_iter != this->_children.end(); grp_iter++) { - grp_iter->second->copyTo(group.groups.groups_val[i++]); - } - } - - return NC_NOERR; -} - -/** - * Return ndims, nvars, natts, and unlimdimp. - */ -int NcGroupInfo::inq( - int *ndimsp, - int *nvarsp, - int *nattsp, - int *unlimdimidp) -{ - int rc = NC_NOERR; - - *ndimsp = this->_dims.size(); - *nvarsp = this->_vars.size(); - *nattsp = this->_atts.size(); - *unlimdimidp = this->_unlimdimid; - - return rc; -} - - - -/** Return the unlimdimid of this group. */ -int NcGroupInfo::inq_unlimdimid(int *unlimdimidp) -{ - int rc = NC_NOERR; - *unlimdimidp = this->_unlimdimid; - return rc; -} - -/** Return the ncid of this group. */ -int NcGroupInfo::inq_ncid(int *grp_ncid) -{ - int rc = NC_NOERR; - *grp_ncid = this->_ncid; - return rc; -} - -/** Return the ncid of a named group */ -int NcGroupInfo::inq_ncid(const char *name, int *grp_ncid) -{ - int rc = NC_NOERR; - return rc; -} - -/** Return the number of groups and copy the ncids of - * each group in a previously allocated array of ints. - * If array is NULL, return the number of groups. - */ -int NcGroupInfo::inq_grps(int *numgrps, int *ncids) -{ - int rc = NC_NOERR; - - *numgrps = _children.size(); - - if (ncids != NULL) { - int count = 0; - map::iterator iter; - - for (iter = _children.begin(); iter != _children.end(); iter++) { - ncids[count++] = iter->first; - } - } - - return rc; -} - -/** Return the length of the dimension (dimid) */ -int NcGroupInfo::inq_dimlen (const int dimid, size_t *lenp) -{ - int rc = NC_NOERR; - - if (_dims.find(dimid) != _dims.end()) { - *lenp = _dims[dimid]->_len; - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -/** Return the name and length of the dimension (dimid) */ -int NcGroupInfo::inq_dim (const int dimid, char *name, size_t *lenp) -{ - int rc = NC_NOERR; - - if (_dims.find(dimid) != _dims.end()) { - *lenp = _dims[dimid]->_len; - if (name != NULL) { - strcpy(name, _dims[dimid]->_name.c_str()); - } - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -/** Return the dimid of the dimension named 'name' */ -int NcGroupInfo::inq_dimid (const char *name, int *dimid) -{ - int rc = NC_EBADDIM; - std::map::iterator iter; - - if (_dims.empty()) { - rc=NC_EBADDIM; - } else { - iter = _dims.begin(); - for (;iter != _dims.end(); iter++) { - if (!strcmp((*iter).second->_name.c_str(), name)) { - *dimid=(*iter).first; - rc = NC_NOERR; - break; - } - } - } - - return rc; -} - -/** Return the dimension IDs */ -int NcGroupInfo::inq_vardimid (const int varid, int dimids[]) -{ - int rc = NC_NOERR; - - if (_vars.find(varid) != _vars.end()) { - NcVarInfo *varInfo = _vars[varid]; - copy(varInfo->_dimids.begin(), varInfo->_dimids.end(), dimids); - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -/** Return the dimension IDs */ -int NcGroupInfo::inq_varndims (const int varid, int *ndimsp) -{ - int rc = NC_NOERR; - *ndimsp = 0; - - if (_vars.find(varid) != _vars.end()) { - *ndimsp = _vars[varid]->_dimids.size(); - } - else { - rc = NC_EEXIST; - } - - return rc; -} - - -/** Add a new variable to this group. */ -#if USE_NC_TYPE -int NcGroupInfo::def_var( - const int varid, - const char *name, - const nc_type xtype, - const int ndims, - const int dimids[]) -#else -int NcGroupInfo::def_var( - const int varid, - const char *name, - const int xtype, - const int ndims, - const int dimids[]) -#endif -{ - int rc = NC_NOERR; - - if (_vars.find(varid) == _vars.end()) { - NcVarInfo *varInfo = new NcVarInfo(varid, name, xtype, ndims, dimids); - _vars[varid] = varInfo; - _varsByName[name] = varInfo; - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -/** Return the number of variables and copy the varids - * of each variable into the previously allocated array - * of ints. If the array is NULL, just return the number - * of variables. - */ -int NcGroupInfo::inq_vars(int *numvars, int *varids) -{ - int rc = NC_NOERR; - map::iterator iter; - - *numvars = this->_vars.size(); - - if (varids != NULL) { - int count = 0; - - for (iter = _vars.begin(); iter != _vars.end(); iter++) { - varids[count++] = iter->first; - } - - assert(*numvars == count); - } - - return rc; -} - -/** Get info about a variable, referenced by varid. */ -#if USE_NC_TYPE -int NcGroupInfo::inq_var ( - const int varid, - char *name, - nc_type *xtypep, - int *ndimsp, - int dimids[], - int *nattsp) -#else -int NcGroupInfo::inq_var ( - const int varid, - char *name, - int *xtypep, - int *ndimsp, - int dimids[], - int *nattsp) -#endif -{ - int rc = NC_NOERR; - - if (_vars.find(varid) != _vars.end()) { - rc = _vars[varid]->inq_var(name, xtypep, ndimsp, dimids, nattsp); - } - else { - rc = NC_ENOTVAR; - } - - return rc; -} - -int NcGroupInfo::inq_varid( - const char *name, - int *varidp) -{ - int rc = NC_NOERR; - - if (_varsByName.find(name) != _varsByName.end()) { - rc = _varsByName[name]->inq_varid(varidp); - } - else { - rc = NC_ENOTVAR; - } - return rc; -} - -/** - * Returns the number of values in a variable by summing - * up the dimension lengths of each dimension. - * - * TODO: cache total dimension sum. - * - * We need to traverse the data structure every time because one of the - * dimensions may not be fixed. There is room for optimization here. - */ -int NcGroupInfo::inq_varsize(const int varid, size_t *countp) -{ - int rc = NC_NOERR; - - *countp = 1; - - if (_vars.find(varid) != _vars.end()) { - uint32_t i; - NcVarInfo *varinfo = _vars[varid]; - - for (i=0; i_dimids.size(); i++) { - int dimid = varinfo->_dimids[i]; - - /* find the dim */ - if (_dims.find(dimid) != _dims.end()) { - NcDimInfo *dim = _dims[dimid]; - *countp *= dim->_len; - } - else { - log_error(netcdf_debug_level, "Could not find dimid=%d", dimid); - rc = NC_ENOTVAR; - goto cleanup; - } - } - } - else { - log_error(netcdf_debug_level, "Could not find varid=%d", varid); - rc = NC_ENOTVAR; - goto cleanup; - } - -cleanup: - return rc; -} - -/** - * Returns the type of a specified variable. - * - */ -#if USE_NC_TYPE -int NcGroupInfo::inq_vartype(const int varid, nc_type *xtypep) -#else -int NcGroupInfo::inq_vartype(const int varid, int *xtypep) -#endif -{ - int rc = NC_NOERR; - - if (_vars.find(varid) != _vars.end()) { - rc = _vars[varid]->inq_vartype(xtypep); - } - - return rc; -} - - - -/** Define a new demension */ -int NcGroupInfo::def_dim( - const int dimid, - const char *name, - const size_t len) -{ - int rc = NC_NOERR; - - if (_dims.find(dimid) == _dims.end()) { - _dims[dimid] = new NcDimInfo(dimid, name, len); - if (len == NC_UNLIMITED) { - this->_unlimdimid = dimid; - } - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -/** Return the dimension ids for all dimensions in a group, or - * any of its parents. If the array is NULL, return the number - * of dimensions defined for all associated groups. - */ -int NcGroupInfo::inq_dims(int *numdims, int *dimids) -{ - int rc = NC_NOERR; - map::iterator iter; - - *numdims = this->_dims.size(); - - if (dimids != NULL) { - int count = 0; - - for (iter = _dims.begin(); iter != _dims.end(); iter++) { - dimids[count++] = iter->first; - } - - assert(*numdims == count); - } - - return rc; -} - -/** - * As records are written in the unlimited dimension, we have to increase - * the length of the unlimited dimension. - */ -int NcGroupInfo::update_unlimdim_dimlen(size_t new_len) -{ - int rc = NC_NOERR; - - if (this->_unlimdimid == -1) { - rc = NC_EBADDIM; - } else { - if (_dims.find(this->_unlimdimid) != _dims.end()) { - if (_dims[this->_unlimdimid]->_len < new_len) { - _dims[this->_unlimdimid]->_len=new_len; - } - } - else { - rc = NC_EBADDIM; - } - } - - return rc; -} - - -/** Copy the name of this group into a previously allocated - * char array. If NULL is passed as the array, return only - * the length of the array. - */ -int NcGroupInfo::inq_name(int *namelen, char *name) -{ - int rc = NC_NOERR; - - *namelen = this->_name.size(); - - if (name != NULL) { - strcpy(name, this->_name.c_str()); - } - - return rc; -} - -/** Return the id of the groups parent. - * - * @returns NC_ENOGRP if this is the root group. - */ -int NcGroupInfo::inq_grp_parent(int *parent_ncid) -{ - int rc = NC_NOERR; - if (this->_parent) { - rc = this->_parent->inq_ncid(parent_ncid); - } - else { - rc = NC_ENOGRP; - } - return rc; -} - -/** Create a new child of this group. - */ -int NcGroupInfo::def_grp(const int new_ncid, const char *name) -{ - int rc = NC_NOERR; - - if (_children.find(new_ncid) == _children.end()) { - _children[new_ncid] = new NcGroupInfo(new_ncid, name, *this); - } - else { - rc = NC_EEXIST; - } - - return rc; -} - - - -/** Global attributes - * - * This family of functions returns information about a netCDF attribute. - * All but one of these functions require the variable ID and attribute - * name; the exception is nc_inq_attname. Information about an attribute - * includes its type, length, name, and number. See the nc_get_att family - * for getting attribute values. - * - * The function inq_attname gets the name of an attribute, given its - * variable ID and number. This function is useful in generic applications - * that need to get the names of all the attributes associated with a - * variable, since attributes are accessed by name rather than number - * in all other attribute functions. The number of an attribute is more - * volatile than the name, since it can change when other attributes of - * the same variable are deleted. This is why an attribute number is not - * called an attribute ID. - * - * The function nc_inq_att returns the attribute's type and length. - * The other functions each return just one item of information about - * an attribute. - */ - -/** Define an attribute for this variable. */ -#if USE_NC_TYPE -int NcGroupInfo::def_att( - const char *name, - const nc_type xtype, - const size_t len) -#else -int NcGroupInfo::def_att( - const char *name, - const int xtype, - const size_t len) -#endif -{ - int rc = NC_NOERR; - - if (_atts.find(name) == _atts.end()) { - _atts[name] = new NcAttInfo(name, xtype, len); - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -#if USE_NC_TYPE -int NcGroupInfo::inq_att( - const int varid, - const char *name, - nc_type *xtypep, - size_t *lenp) -#else -int NcGroupInfo::inq_att( - const int varid, - const char *name, - int *xtypep, - size_t *lenp) -#endif -{ - int rc = NC_NOERR; - - if (varid == NC_GLOBAL) { - if (_atts.find(name) != _atts.end()) { - _atts[name]->inq_atttype(xtypep); - _atts[name]->inq_attlen(lenp); - } - else { - rc = NC_ENOTATT; - } - } - - else { - if (_vars.find(varid) != _vars.end()) { - _vars[varid]->inq_att(name, xtypep, lenp); - } - else { - rc = NC_ENOTVAR; - } - } - return rc; -} - - -#if USE_NC_TYPE -int NcGroupInfo::inq_atttype( - const int varid, - const char *name, - nc_type *xtypep) -#else -int NcGroupInfo::inq_atttype( - const int varid, - const char *name, - int *xtypep) -#endif -{ - int rc = NC_NOERR; - - if (varid == NC_GLOBAL) { - if (_atts.find(name) != _atts.end()) { - _atts[name]->inq_atttype(xtypep); - } - else { - rc = NC_ENOTATT; - } - } - - else { - if (_vars.find(varid) != _vars.end()) { - _vars[varid]->inq_atttype(name, xtypep); - } - else { - rc = NC_ENOTVAR; - } - } - return rc; -} - - -int NcGroupInfo::inq_attlen( - const int varid, - const char *name, - size_t *lenp) -{ - int rc = NC_NOERR; - - if (varid == NC_GLOBAL) { - if (_atts.find(name) != _atts.end()) { - _atts[name]->inq_attlen(lenp); - } - else { - rc = NC_ENOTATT; - } - } - - else { - if (_vars.find(varid) != _vars.end()) { - _vars[varid]->inq_attlen(name, lenp); - } - else { - rc = NC_ENOTVAR; - } - } - return rc; -} - -int NcGroupInfo::inq_attname( - const int varid, - const int attnum, - char *name) -{ - int rc = NC_NOERR; - - if (varid == NC_GLOBAL) { - std::map::iterator iter; - - iter=_atts.begin(); - for (int i=0;iinq_attname(name); - } - - else { - if (_vars.find(varid) != _vars.end()) { - _vars[varid]->inq_attname(attnum, name); - } - else { - rc = NC_ENOTVAR; - } - } - return rc; -} - -int NcGroupInfo::inq_attid( - const int varid, - const char *name, - int *attnump) -{ - int rc = NC_ENOTSUPP; - return rc; -} - - - -/** Delete an attribute. */ -int NcGroupInfo::del_att (const char* name) -{ - int rc = NC_NOERR; - return rc; -} - -#endif // HAVE_TRIOS_PNETCDF diff --git a/packages/trios/services/netcdf/src/NcGroupInfo.h b/packages/trios/services/netcdf/src/NcGroupInfo.h deleted file mode 100644 index d27ee045a88f..000000000000 --- a/packages/trios/services/netcdf/src/NcGroupInfo.h +++ /dev/null @@ -1,320 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncGroup.h - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#ifndef NCGROUP_H_ -#define NCGROUP_H_ - -#include -#include - -#include "NcFileInfo.h" - -class NcDimInfo; -class NcAttInfo; -class NcVarInfo; - -#undef USE_NC_TYPE -#define USE_NC_TYPE 1 - -/** - * The ncGroup represent a netCDF group using the netCDF-4 definition. - * - * Excerpt from netcdf page... - * - * Groups are identified with a ncid, which identifies both the open file, and the - * group within that file. When a file is opened with nc_open or nc_create, - * the ncid for the root group of that file is provided. Using that as a - * starting point, users can add new groups, or list and navigate existing groups. - * - * All netCDF calls take a ncid which determines where the call will take its - * action. For example, the nc_def_var function takes a ncid as its first parameter. - * It will create a variable in whichever group its ncid refers to. Use the root - * ncid provided by nc_create or nc_open to create a variable in the root group. - * Or use nc_def_grp to create a group and use its ncid to define a variable in - * the new group. - * - * Variable are only visible in the group in which they are defined. The - * same applies to attributes. �Global� attributes are associated with - * the group whose ncid is used. - * - * Dimensions are visible in their groups, and all child groups. - * - * Group operations are only permitted on netCDF-4 files - that is, - * files created with the HDF5 flag in nc_create. (see nc_create). - * Groups are not compatible with the netCDF classic data model, so - * files created with the NC_CLASSIC_MODEL file cannot contain groups - * (except the root group). - */ - -class NcGroupInfo { - -public: - - /** Create a new root group */ - NcGroupInfo(const int ncid, const NcFileInfo &fileInfo); - - - /** Create a new child group. */ - NcGroupInfo(const int ncid, const char *name, NcGroupInfo &parent); - - /** Create a new child group from the info in struct nc_group. */ - NcGroupInfo(const struct nc_group &group, NcGroupInfo &parent); - - - /** Create a root group from the struct nc_group. */ - NcGroupInfo(const struct nc_group &group, const NcFileInfo &fileInfo); - - - virtual ~NcGroupInfo(); - - /** Construct from a struct nc_group */ - NcGroupInfo(const struct nc_group &group, NcFileInfo &fileInfo); - - /* copy contents from struct nc_group */ - int copyFrom(const struct nc_group &group); - - /* Convert to struct nc_group */ - int copyTo(struct nc_group &group); - - /** Add a new sub-group. */ - int def_grp(const int child_ncid, const char *name); - - /** Return the ncid of this group. */ - int inq_ncid(int *ncid); - - /** Return the ncid of a named child group. */ - int inq_ncid(const char *name, int *ncid); - - /** Return the number of groups and copy the ncids of - * each group in a previously allocated array of ints. - * If array is NULL, return the number of groups. - */ - int inq_grps(int *numgrps, int *ncids); - - /** Copy the name of this group into a previously allocated - * char array. If NULL is passed as the array, return only - * the length of the array. - */ - int inq_name(int *namelen, char *name); - - /** Return the id of the groups parent. - * - * @returns NC_ENOGRP if this is the root group. - */ - int inq_grp_parent(int *parent_ncid); - - - /* --------- VARIABLES --------------- */ - - /** Add a new variable to this group. */ -#if USE_NC_TYPE - int def_var(const int varid, const char *name, - const nc_type xtype, const int ndims, const int dimids[]); -#else - int def_var(const int varid, const char *name, - const int xtype, const int ndims, const int dimids[]); -#endif - - /** Delete a variable. */ - int del_var(const int varid); - - /** Return the number of variables and copy the varids - * of each variable into the previously allocated array - * of ints. If the array is NULL, just return the number - * of variables. - */ - int inq_vars(int *numvars, int *varids); - - /** Get info about a variable, referenced by varid. */ - int inq_var (const int varid, char *name, nc_type *xtypep, - int *ndimsp, int dimids[], int *nattsp); - - /** Get the ID given a variable name. */ - int inq_varid (const char *name, int *varidp); - - /** Get name of a variable. */ - int inq_varname (const int varid, char *name); - - /** Get type of variable. */ - int inq_vartype (const int varid, nc_type *xtypep); - - /** Get number of dimensions used by variable. */ - int inq_varndims (const int varid, int *ndimsp); - - /** Return the total number of values in a variable. */ - int inq_varsize(const int varid, size_t *sizep); - - /** Get dimids used by variable. */ - int inq_vardimid (const int varid, int dimids[]); - - /** Get number of attributes used by a variable. */ - int inq_varnatts (const int varid, int *nattsp); - - /* -------- DIMENSIONS -------------- */ - - /** Define a new demension */ - int def_dim(const int dimid, const char *name, const size_t len); - - /** Add a new dimension. */ - - /** Rename a dimension. */ - int rename_dim(const int dimid, const char *name); - - /** General inquiry */ - int inq(int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimp); - - /** Find unlimited dimension IDs. */ - int inq_unlimdims(int *nunlimdimsp, int *unlimdimidsp); - - /** Return the dimension ids for all dimensions in a group, or - * any of its parents. If the array is NULL, return the number - * of dimensions defined for all associated groups. - */ - int inq_dims(int *numdims, int *dimids); - - /** - * As records are written in the unlimited dimension, we have to increase - * the length of the unlimited dimension. - */ - int update_unlimdim_dimlen(size_t new_len); - - /** Return the name and length of a dimension referenced by dimid. */ - int inq_dim(const int dimid, char *name, size_t *lengthp); - - /** Return the name of a dimension. */ - int inq_dimname(const int dimid, char *name); - - /** Return the length of a dimension. */ - int inq_dimlen(const int dimid, size_t *lengthp); - - /** Return the dimid of the dimension named 'name' */ - int inq_dimid (const char *name, int *dimid); - - int inq_unlimdimid(int *unlimdimidp); - - - /** Attributes - * - * This family of functions returns information about a netCDF attribute. - * All but one of these functions require the variable ID and attribute - * name; the exception is nc_inq_attname. Information about an attribute - * includes its type, length, name, and number. See the nc_get_att family - * for getting attribute values. - * - * The function inq_attname gets the name of an attribute, given its - * variable ID and number. This function is useful in generic applications - * that need to get the names of all the attributes associated with a - * variable, since attributes are accessed by name rather than number - * in all other attribute functions. The number of an attribute is more - * volatile than the name, since it can change when other attributes of - * the same variable are deleted. This is why an attribute number is not - * called an attribute ID. - * - * The function nc_inq_att returns the attribute's type and length. - * The other functions each return just one item of information about - * an attribute. - */ -#if USE_NC_TYPE - int inq_att (const int varid, const char *name, - nc_type *xtypep, size_t *lenp); - int inq_atttype(const int varid, const char *name, - nc_type *xtypep); -#else - int inq_att (const int varid, const char *name, - int *xtypep, size_t *lenp); - int inq_atttype(const int varid, const char *name, - int *xtypep); -#endif - int inq_attlen (const int varid, const char *name, size_t *lenp); - int inq_attname(const int varid, int attnum, char *name); - int inq_attid (const int varid, const char *name, int *attnump); - - /** Define an attribute for this variable. */ -#if USE_NC_TYPE - int def_att(const char *name, const nc_type xtype, const size_t len); -#else - int def_att(const char *name, const int xtype, const size_t len); -#endif - - /** Delete an attribute. */ - int del_att (const char* name); - - -protected: - - - /** Construct a child group */ - NcGroupInfo(const int ncid, const char *name, NcGroupInfo *parent); - - - -private: - const int _ncid; /* id of this group */ - const NcFileInfo _fileInfo; /* points to root group (null if we are the root) */ - NcGroupInfo *_parent; /* pointer to parent (null if we are the root) */ - std::string _name; /* name of this group "/" if root */ - int _unlimdimid; - - -public: - std::map _vars; /* variables used by the group */ - std::map _varsByName; /* variables mapped by name */ - std::map _dims; /* Dimensions used by this group */ - - std::map _atts; /* global attributes for this group */ - - std::map _children; /* Subgroups of this group */ -}; - - -/** Global map of open groups (used by ncmpi_client and netcdf_client) */ -extern map group_map; - - -#endif /* NCGROUP_H_ */ diff --git a/packages/trios/services/netcdf/src/NcVarInfo.cpp b/packages/trios/services/netcdf/src/NcVarInfo.cpp deleted file mode 100644 index 3dc8e7e37c85..000000000000 --- a/packages/trios/services/netcdf/src/NcVarInfo.cpp +++ /dev/null @@ -1,425 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncVar.cpp - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#include "Trios_config.h" -#ifdef HAVE_TRIOS_PNETCDF - -#include -#include -using namespace std; - -#include -#include -#include "NcVarInfo.h" -#include "NcAttInfo.h" -#include "netcdf_debug.h" - - - -#if USE_NC_TYPE -NcVarInfo::NcVarInfo( - const int varid, - const char *name, - const nc_type xtype, - const int ndims, - const int dimids[]) : - _varid(varid), _name(name), _xtype(xtype), _dimids(dimids, dimids+ndims) -#else -NcVarInfo::NcVarInfo( - const int varid, - const char *name, - const int xtype, - const int ndims, - const int dimids[]) : - _varid(varid), _name(name), _xtype(xtype), _dimids(dimids, dimids+ndims) -#endif -{ } - -NcVarInfo::NcVarInfo(const nc_var &var) : - _varid(var.varid), _name(var.name), _xtype((nc_type)var.xtype), - _dimids(var.dimids.dimids_val, var.dimids.dimids_val + var.dimids.dimids_len) -{ - for (uint32_t i=0;i_atts[var.atts.atts_val[i].name] = new NcAttInfo(var.atts.atts_val[i]); - } -} - - - -NcVarInfo::~NcVarInfo() -{ -} - -/** - * Convert to struct nc_var. - */ -int NcVarInfo::copyTo(struct nc_var &var) -{ - int natts, ndims; - log_level debug_level = netcdf_debug_level; - - memset(&var, 0, sizeof(struct nc_var)); - - var.name = strdup(this->_name.c_str()); - var.varid = this->_varid; - var.xtype = this->_xtype; - - /* copy attributes */ - natts = this->_atts.size(); - var.atts.atts_len = natts; - log_debug(debug_level, "copy %d atts", natts); - if (natts) { - map::iterator att_iter; - int i=0; - var.atts.atts_val = (struct nc_att *)calloc(natts, sizeof(struct nc_att)); - for (att_iter = this->_atts.begin(); att_iter != this->_atts.end(); att_iter++) { - att_iter->second->copyTo(var.atts.atts_val[i++]); - } - } - - /* copy dimids */ - ndims = this->_dimids.size(); - var.dimids.dimids_len = ndims; - log_debug(debug_level, "copy %d dimids", ndims); - if (ndims) { - int i=0; - vector::iterator dim_iter; - var.dimids.dimids_val = (int *)calloc(ndims, sizeof(int)); - for (dim_iter = this->_dimids.begin(); dim_iter != this->_dimids.end(); dim_iter++) { - var.dimids.dimids_val[i++] = *dim_iter; - } - } - - return NC_NOERR; -} - - -/** Get the variable ID. */ -int NcVarInfo::inq_varid(int *varidp) -{ - int rc = NC_NOERR; - *varidp = this->_varid; - return rc; -} - - -/** Get information about a variable. */ -#if USE_NC_TYPE -int NcVarInfo::inq_var(char *name, nc_type *xtypep, int *ndimsp, - int dimids[], int *nattsp) -#else -int NcVarInfo::inq_var(char *name, int *xtypep, int *ndimsp, - int dimids[], int *nattsp) -#endif -{ - int rc = NC_NOERR; - if (name != NULL) { - strcpy(name, this->_name.c_str()); - } - - *xtypep = this->_xtype; - *ndimsp = this->_dimids.size(); - - if (dimids != NULL) { - std::copy(this->_dimids.begin(), this->_dimids.end(), dimids); - } - - *nattsp = this->_atts.size(); - - return rc; -} - - -/** Get name of variable. */ -int NcVarInfo::inq_varname(char *name) -{ - int rc = NC_NOERR; - - if (name != NULL) { - strcpy(name, this->_name.c_str()); - } - - return rc; -} - -/** Get type of variable. */ -#if USE_NC_TYPE -int NcVarInfo::inq_vartype(nc_type *xtypep) -#else -int NcVarInfo::inq_vartype(int *xtypep) -#endif -{ - int rc = NC_NOERR; - *xtypep = this->_xtype; - return rc; -} - -/** Get the number of dimensions used for this variable. */ -int NcVarInfo::inq_varndims(int *ndimsp) -{ - int rc = NC_NOERR; - *ndimsp = this->_dimids.size(); - return rc; -} - -/** Get the dimension ids for this variable. */ -int NcVarInfo::inq_vardimid(int dimids[]) -{ - int rc = NC_NOERR; - if (dimids != NULL) { - std::copy(this->_dimids.begin(), this->_dimids.end(), dimids); - } - return rc; -} - - -/** Get the number of attributes for this variable. */ -int NcVarInfo::inq_varnatts(int *nattsp) -{ - int rc = NC_NOERR; - *nattsp = this->_atts.size(); - return rc; -} - - -/** Set the chunking parameters for netCDF-4 files. */ -int NcVarInfo::def_var_chunking(const int contiguous, int *chunksizep) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Inquire about chunking paramenters for this variable. */ -int NcVarInfo::inq_var_chunking(int *contiguousp, int *chunksizep) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Define fill parameters for a variable. */ -int NcVarInfo::def_var_fill(const int no_fill, void *fill_value) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Inquire about fill parameters. */ -int NcVarInfo::inq_var_fill(int *no_fill, void *fill_value) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Define compression parameters. */ -int NcVarInfo::def_var_deflate(const int shuffle, const int deflate, - const int deflate_level) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Inquire about compression parameters. */ -int NcVarInfo::inq_var_deflate(int *shufflep, int *deflatep, int *deflate_levelp) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Define fletcher32 parameters for netcdf-4 variable. */ -int NcVarInfo::def_var_fletcher32(const int fletcher32) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Inquire about fletcher32 parameter. */ -int NcVarInfo::inq_var_fletcher32(int *fletcher32p) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Define endianness of variable. NC_ENDIAN_NATIVE, NC_ENDIAN_LITTLE, NC_ENDIAN_BIG */ -int NcVarInfo::def_var_endian(const int endian) -{ - int rc = NC_ENOTSUPP; - return rc; -} - -/** Inquire about endiannes of variable. */ -int NcVarInfo::inq_var_endian(int *endianp) -{ - int rc = NC_ENOTSUPP; - return rc; -} - - - -/** Get attributes for a variable. - * - * This family of functions returns information about a netCDF attribute. - * All but one of these functions require the variable ID and attribute - * name; the exception is nc_inq_attname. Information about an attribute - * includes its type, length, name, and number. See the nc_get_att family - * for getting attribute values. - * - * The function inq_attname gets the name of an attribute, given its - * variable ID and number. This function is useful in generic applications - * that need to get the names of all the attributes associated with a - * variable, since attributes are accessed by name rather than number - * in all other attribute functions. The number of an attribute is more - * volatile than the name, since it can change when other attributes of - * the same variable are deleted. This is why an attribute number is not - * called an attribute ID. - * - * The function nc_inq_att returns the attribute's type and length. - * The other functions each return just one item of information about - * an attribute. - */ -/** Define an attribute for this variable. */ -#if USE_NC_TYPE -int NcVarInfo::def_att( - const char *name, - const nc_type xtype, - const size_t len) -#else -int NcVarInfo::def_att( - const char *name, - const int xtype, - const size_t len) -#endif -{ - int rc = NC_NOERR; - - if (_atts.find(name) == _atts.end()) { - _atts[name] = new NcAttInfo(name, xtype, len); - } - else { - rc = NC_EEXIST; - } - - return rc; -} - -#if USE_NC_TYPE -int NcVarInfo::inq_att (const char *name, - nc_type *xtypep, size_t *lenp) -#else -int NcVarInfo::inq_att (const char *name, - int *xtypep, size_t *lenp) -#endif -{ - int rc = NC_NOERR; - - if (_atts.find(name) != _atts.end()) { - _atts[name]->inq_atttype(xtypep); - _atts[name]->inq_attlen(lenp); - } - else { - rc = NC_ENOTATT; - } - - return rc; -} - -#if USE_NC_TYPE -int NcVarInfo::inq_atttype(const char *name, - nc_type *xtypep) -#else -int NcVarInfo::inq_atttype(const char *name, - int *xtypep) -#endif -{ - int rc = NC_NOERR; - if (_atts.find(name) != _atts.end()) { - _atts[name]->inq_atttype(xtypep); - } - else { - rc = NC_ENOTATT; - } - return rc; -} - - -int NcVarInfo::inq_attlen (const char *name, size_t *lenp) -{ - int rc = NC_NOERR; - if (_atts.find(name) != _atts.end()) { - _atts[name]->inq_attlen(lenp); - } - else { - rc = NC_ENOTATT; - } - return rc; -} - - -int NcVarInfo::inq_attname(int attnum, char *name) -{ - int rc = NC_NOERR; - std::map::iterator iter; - - iter=_atts.begin(); - for (int i=0;iinq_attname(name); - - return rc; -} - - -int NcVarInfo::inq_attid (const char *name, int *attnump) -{ - int rc = NC_ENOTSUPP; - return rc; -} - - -#endif // HAVE_TRIOS_PNETCDF - diff --git a/packages/trios/services/netcdf/src/NcVarInfo.h b/packages/trios/services/netcdf/src/NcVarInfo.h deleted file mode 100644 index 273fdba644f9..000000000000 --- a/packages/trios/services/netcdf/src/NcVarInfo.h +++ /dev/null @@ -1,243 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * ncVar.h - * - * Created on: Jan 22, 2009 - * Author: raoldfi - */ - -#ifndef NCVAR_H_ -#define NCVAR_H_ - -#include -#include -#include - -using namespace std; - -#include "netcdf_args.h" - - -class NcFileInfo; -class NcAttInfo; - -/* whether or not to use the nc_type since netcdf and pnetcdf conflict */ -#undef USE_NC_TYPE -#define USE_NC_TYPE 1 - -class NcVarInfo { - -public: - - /** Convert to struct nc_var */ - int copyTo(struct nc_var &); - - /** Get the variable ID. */ - int inq_varid(int *varidp); - - /** Get information about a variable. */ -#if USE_NC_TYPE - int inq_var(char *name, nc_type *xtypep, int *ndmisp, - int dimids[], int *nattsp); -#else - int inq_var(char *name, int *xtypep, int *ndmisp, - int dimids[], int *nattsp); -#endif - - /** Get name of variable. */ - int inq_varname(char *name); - - /** Get type of variable. */ -#if USE_NC_TYPE - int inq_vartype(nc_type *xtypep); -#else - int inq_vartype(int *xtypep); -#endif - - /** Get the number of dimensions used for this variable. */ - int inq_varndims(int *ndimsp); - - /** Get the dimension ids for this variable. */ - int inq_vardimid(int dimids[]); - - /** Get the number of attributes for this variable. */ - int inq_varnatts(int *nattsp); - - - /** Set the chunking parameters for netCDF-4 files. */ - int def_var_chunking(const int contiguous, int *chunksizep); - - /** Inquire about chunking paramenters for this variable. */ - int inq_var_chunking(int *contiguousp, int *chunksizep); - - /** Define fill parameters for a variable. */ - int def_var_fill(const int no_fill, void *fill_value); - - /** Inquire about fill parameters. */ - int inq_var_fill(int *no_fill, void *fill_value); - - /** Define compression parameters. */ - int def_var_deflate(const int shuffle, const int deflate, - const int deflate_level); - - /** Inquire about compression parameters. */ - int inq_var_deflate(int *shufflep, int *deflatep, int *deflate_levelp); - - /** Define fletcher32 parameters for netcdf-4 variable. */ - int def_var_fletcher32(const int fletcher32); - - /** Inquire about fletcher32 parameter. */ - int inq_var_fletcher32(int *fletcher32p); - - /** Define endianness of variable. NC_ENDIAN_NATIVE, NC_ENDIAN_LITTLE, NC_ENDIAN_BIG */ - int def_var_endian(const int endian); - - /** Inquire about endiannes of variable. */ - int inq_var_endian(int *endianp); - - - - /** Variable attributes - * - * This family of functions returns information about a netCDF attribute. - * All but one of these functions require the variable ID and attribute - * name; the exception is nc_inq_attname. Information about an attribute - * includes its type, length, name, and number. See the nc_get_att family - * for getting attribute values. - * - * The function inq_attname gets the name of an attribute, given its - * variable ID and number. This function is useful in generic applications - * that need to get the names of all the attributes associated with a - * variable, since attributes are accessed by name rather than number - * in all other attribute functions. The number of an attribute is more - * volatile than the name, since it can change when other attributes of - * the same variable are deleted. This is why an attribute number is not - * called an attribute ID. - * - * The function nc_inq_att returns the attribute's type and length. - * The other functions each return just one item of information about - * an attribute. - */ -#if USE_NC_TYPE - int inq_att (const char *name, - nc_type *xtypep, size_t *lenp); - int inq_atttype(const char *name, - nc_type *xtypep); -#else - int inq_att (const char *name, - int *xtypep, size_t *lenp); - int inq_atttype(const char *name, - int *xtypep); -#endif - int inq_attlen (const char *name, size_t *lenp); - int inq_attname(int attnum, char *name); - int inq_attid (const char *name, int *attnump); - - /** Define an attribute for this variable. */ -#if USE_NC_TYPE - int def_att(const char *name, const nc_type xtype, const size_t len); -#else - int def_att(const char *name, const int xtype, const size_t len); -#endif - - /** Delete an attribute. */ - int del_att (const char* name); - - - -public: - - /** Create a new variable for a netcdf dataset. */ -#if USE_NC_TYPE - NcVarInfo(const int varid, const char *name, const nc_type xtype, - const int ndims, const int dimids[]); -#else - NcVarInfo(const int varid, const char *name, const int xtype, - const int ndims, const int dimids[]); -#endif - - NcVarInfo(const struct nc_var &var); - - virtual ~NcVarInfo(); - - -private: - const int _varid; - - string _name; -#if USE_NC_TYPE - const nc_type _xtype; -#else - const int _xtype; -#endif - - - /* chunking parameters */ - int _contiguous; - int _chunksize; - - /* fill parameters */ - int _no_fill; - void *_fill_value; - - /* compression parameters */ - int _shuffle; - int _deflate; - int _deflate_level; - - /* fletcher32 parameters */ - int _fletcher32; - -public: - - /* Map of attributes, indexed by name. */ - map _atts; - - /* Vector of dimension IDs used by this variable. The actual - * dimensions are stored in the NcDataset structure for this ncid. - */ - vector _dimids; /* dimension IDS for this variable */ -}; - -#endif /* NCVAR_H_ */ diff --git a/packages/trios/services/netcdf/src/aggregation.cpp b/packages/trios/services/netcdf/src/aggregation.cpp deleted file mode 100644 index 0dea178a35f4..000000000000 --- a/packages/trios/services/netcdf/src/aggregation.cpp +++ /dev/null @@ -1,1071 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * aggregation.cpp - * - * Created on: Mar 9, 2009 - * Author: thkorde - */ - -#include -#include - -#include /* for nssi service functions */ - -#include "pnetcdf.h" -#include -#include -#include -#include - -using namespace std; - -#include "netcdf_args.h" -#include "netcdf_debug.h" /* netcdf_debug_level */ - -#include "aggregation.h" - -typedef struct { - int ahead_count; - int behind_count; - int same_count; - int no_match_count; -} chunk_location_count_t; - -struct participant_t { - NNTI_peer_t p; -}; -typedef struct participant_t participant_t; - -typedef list participants_t; -typedef list::iterator participants_iterator_t; - -typedef list chunk_details_t; -typedef list::iterator chunk_details_iterator_t; - -typedef struct { - aggregation_chunk_details_t *details; - chunk_details_t component_chunks; -} aggregation_chunk_t; - -typedef list chunks_t; -typedef list::iterator chunks_iterator_t; - - -typedef struct { - int varid; - chunks_t *chunks; - int chunks_received; -} per_var_details_t; - -typedef map nc_var_map_t; -typedef map::iterator nc_var_map_iterator_t; -typedef pair nc_var_map_pair_t; - -typedef struct { - int ncid; - participants_t *participants; - int num_participants; /* will be init to 0. if > 0, then someone forced a specific number of participants. */ - nc_var_map_t nc_vars; - write_type type; /* direct, aggregate independent, aggregate collective */ -} file_details_t; - -static map open_file_map; -typedef map::iterator open_file_map_iterator_t; -typedef pair open_file_map_pair_t; - -bool compare_chunks_for_aggregation(const aggregation_chunk_t* c1, const aggregation_chunk_t* c2) -{ - aggregation_chunk_details_t *details1=c1->details; - aggregation_chunk_details_t *details2=c2->details; - - for (int i=0;indims;i++) { - if (details1->count[i] < details2->count[i]) { - return true; - } else if (details1->count[i] > details2->count[i]) { - return false; - } - } - for (int i=0;indims;i++) { - if (details1->start[i] < details2->start[i]) { - return true; - } else if (details1->start[i] > details2->start[i]) { - return false; - } - } - return false; -} - -bool compare_chunks_for_caching(const aggregation_chunk_t* c1, const aggregation_chunk_t* c2) -{ - aggregation_chunk_details_t *details1=c1->details; - aggregation_chunk_details_t *details2=c2->details; - - for (int i=0;indims;i++) { - if (details1->start[i] < details2->start[i]) { - return true; - } else if (details1->start[i] > details2->start[i]) { - return false; - } - } - return false; -} - -file_details_t *new_open_file(const int ncid) -{ - file_details_t *details=NULL; - - details = new file_details_t; - details->ncid = ncid; - details->participants = new participants_t; - details->num_participants=0; - - return details; -} - -int use_aggregation(const int ncid) -{ - file_details_t *details=NULL; - - details = open_file_map[ncid]; - if ((details->type == WRITE_AGGREGATE_INDEPENDENT) || - (details->type == WRITE_AGGREGATE_COLLECTIVE)) { - return TRUE; - } - - return FALSE; -} - -int use_caching(const int ncid) -{ - file_details_t *details=NULL; - - details = open_file_map[ncid]; - if ((details->type == WRITE_CACHING_INDEPENDENT) || - (details->type == WRITE_CACHING_COLLECTIVE)) { - return TRUE; - } - - return FALSE; -} - -int use_collective(const int ncid) -{ - file_details_t *details=NULL; - - details = open_file_map[ncid]; - if ((details->type == WRITE_AGGREGATE_COLLECTIVE) || - (details->type == WRITE_CACHING_COLLECTIVE)) { - return TRUE; - } - - return FALSE; -} - -int use_independent(const int ncid) -{ - file_details_t *details=NULL; - - details = open_file_map[ncid]; - if ((details->type == WRITE_AGGREGATE_INDEPENDENT) || - (details->type == WRITE_CACHING_INDEPENDENT)) { - return TRUE; - } - - return FALSE; -} - -int use_direct(const int ncid) -{ - file_details_t *details=NULL; - - details = open_file_map[ncid]; - if (details->type == WRITE_DIRECT) { - return TRUE; - } - - return FALSE; -} - -void add_participant_for_file(const int ncid, - const NNTI_peer_t *caller, - const write_type requested_write_type) -{ - file_details_t *details =NULL; - participant_t *participant=NULL; - - log_debug(netcdf_debug_level, "adding participant for ncid=%d", ncid); - details = open_file_map[ncid]; - if (details == NULL) { - log_debug(netcdf_debug_level, "creating new file_details_t for ncid=%d", ncid); - details=new_open_file(ncid); - log_debug(netcdf_debug_level, "new file_details_t=%p", details); - open_file_map[ncid]=details; - } - - participant=new participant_t(); - participant->p=*caller; - details->participants->push_back(participant); - - details->type = requested_write_type; -} - -void set_participant_count(const int ncid, - const NNTI_peer_t *caller, - const write_type requested_write_type, - const int num_participants) -{ - file_details_t *details =NULL; - participant_t *participant=NULL; - - log_debug(netcdf_debug_level, "setting participant count for ncid=%d", ncid); - details = open_file_map[ncid]; - if (details == NULL) { - log_debug(netcdf_debug_level, "creating new file_details_t for ncid=%d", ncid); - details=new_open_file(ncid); - log_debug(netcdf_debug_level, "new file_details_t=%p", details); - open_file_map[ncid]=details; - } - - participant=new participant_t(); - participant->p=*caller; - details->participants->push_back(participant); - details->type = requested_write_type; - details->num_participants=num_participants; -} - -void remove_participant_for_file(const int ncid, const NNTI_peer_t *caller) -{ - file_details_t *details =NULL; - participants_iterator_t iter; - - log_level debug_level = netcdf_debug_level; - - details = open_file_map[ncid]; - if (details == NULL) { - log_error(debug_level, "remove failed - no participants"); - goto out; - } - -out: - return; -} - -void add_participant_chunk(const NNTI_peer_t *caller, - aggregation_chunk_details_t *chunk_details) -{ - file_details_t *file_details=NULL; - - log_debug(netcdf_debug_level, "adding chunk: ncid(%d) varid(%d)", chunk_details->ncid, chunk_details->varid); - - chunk_details->datatype_size=0; - MPI_Type_size(chunk_details->datatype, &chunk_details->datatype_size); - if ((chunk_details->len/chunk_details->num_elements) != (size_t)chunk_details->datatype_size) { - log_warn(netcdf_debug_level, "datatype size conflict: (%d/%d)==%d is not equal to %d", - chunk_details->len, chunk_details->num_elements, chunk_details->len/chunk_details->num_elements, chunk_details->datatype_size); - } - - file_details = open_file_map[chunk_details->ncid]; - if (file_details == NULL) { - log_error(netcdf_debug_level, "failed to add chunk. cannot aggregate."); - return; - } - per_var_details_t *var_details = file_details->nc_vars[chunk_details->varid]; - if (var_details == NULL) { - var_details=new per_var_details_t; - var_details->varid = chunk_details->varid; - var_details->chunks = new chunks_t; - var_details->chunks_received=0; - file_details->nc_vars[chunk_details->varid]=var_details; - } - aggregation_chunk_t *chunk=new aggregation_chunk_t; - chunk->details = chunk_details; - var_details->chunks->push_back(chunk); - var_details->chunks_received++; - - return; -} - -void destroy_chunk(aggregation_chunk_details_t *details) -{ - free(details->start); - free(details->count); - free(details->stride); - log_debug(netcdf_debug_level, "freeing details->buf(%p)", details->buf); - free(details->buf); - delete details; -} - -void cleanup_aggregation_chunks(const int ncid) -{ - file_details_t *details=NULL; - nc_var_map_iterator_t var_iter; - per_var_details_t *var_details=NULL; - - log_debug(netcdf_debug_level, "entered"); - log_debug(netcdf_debug_level, "cleaning up - ncid(%d)", ncid); - - details = open_file_map[ncid]; - if (details == NULL) { - return; - } - var_iter = details->nc_vars.begin(); - for (; var_iter != details->nc_vars.end(); ++var_iter) { - var_details = var_iter->second; - if (!var_details) { - continue; - } - cleanup_aggregation_chunks(ncid, var_details->varid); - } -} - -void cleanup_aggregation_chunks(const int ncid, const int varid) -{ - file_details_t *details=NULL; - aggregation_chunk_t *chunk=NULL; - chunks_iterator_t chunks_iter; - chunk_details_iterator_t component_iter; - nc_var_map_iterator_t vars_iter; - - log_debug(netcdf_debug_level, "cleaning up - ncid(%d) varid(%d)", ncid, varid); - - // for each variable, iterate over the chunks and destroy them - - details = open_file_map[ncid]; - - per_var_details_t *var_details = details->nc_vars[varid]; - if (var_details != NULL) { - chunks_iter = var_details->chunks->begin(); - for (;chunks_iter != var_details->chunks->end(); ++chunks_iter) { - chunk = *chunks_iter; - component_iter = chunk->component_chunks.begin(); - for (;component_iter != chunk->component_chunks.end(); ++component_iter) { - log_debug(netcdf_debug_level, "cleanup - destroying component"); - destroy_chunk(*component_iter); - } - chunk->component_chunks.clear(); - log_debug(netcdf_debug_level, "cleanup - destroying details"); - destroy_chunk(chunk->details); - delete chunk; - } - var_details->chunks->clear(); - var_details->chunks_received=0; - } -} - -static void recursive_print_chunk(aggregation_chunk_details_t *details, int offset, int *index, int current_dim) -{ - int my_offset=0; - char tmp_str[20]; - char out_str[1024]; - int remaining=1023; - - if (current_dim < details->ndims-1) { - for (int i=0;icount[current_dim];i++) { - my_offset = index[current_dim]; - for (int j=current_dim+1;jndims;j++) { - my_offset *= details->count[j]; - } - - index[current_dim+1]=0; - recursive_print_chunk(details, offset+my_offset, index, current_dim+1); - index[current_dim] += details->datatype_size; - } - log_debug(netcdf_debug_level, "-----------------------------"); - } else { - if (details->buf == NULL) { - log_debug(netcdf_debug_level, "details->buf == NULL"); - } else { - out_str[0]='\0'; - for (int i=0;icount[current_dim];i++) { - my_offset = offset+index[current_dim]; - -// if (i==0) log_debug(netcdf_debug_level, "[%d][%d][%d] (my_offset==%d)", index[0], index[1], index[2], my_offset); - if (details->datatype == MPI_BYTE || details->datatype == MPI_CHAR) { - sprintf(tmp_str, "%c, ", *(char *)(((char *)details->buf) + my_offset)); - } - else if (details->datatype == MPI_SHORT) { - sprintf(tmp_str, "%hx, ", *(short *)(((char *)details->buf) + my_offset)); - } - else if (details->datatype == MPI_INT) { - sprintf(tmp_str, "%x, ", *(int *)(((char *)details->buf) + my_offset)); - } - else if (details->datatype == MPI_FLOAT) { - sprintf(tmp_str, "%f, ", *(float *)(((char *)details->buf) + my_offset)); - } - else if (details->datatype == MPI_DOUBLE) { - sprintf(tmp_str, "%f, ", *(double *)(((char *)details->buf) + my_offset)); - } - strncat(out_str, tmp_str, remaining); - remaining -= strlen(out_str); - - index[current_dim] += details->datatype_size; - } -// log_debug(netcdf_debug_level, "[%d][%d][%d] (my_offset==%d)", index[0], index[1], index[2], my_offset); - log_debug(netcdf_debug_level, "%s", out_str); - } - } -} - -void print_chunk(aggregation_chunk_details_t *details) -{ - int *index=(int *)calloc(details->ndims, sizeof(int)); - char tmp_str[20]; - char out_str[1024]; - int remaining=1023; - - log_debug(netcdf_debug_level, "+++++++++++++++++++++++++++++"); - - log_debug(netcdf_debug_level, "ncid==%d", details->ncid); - log_debug(netcdf_debug_level, "varid==%d", details->varid); - log_debug(netcdf_debug_level, "ndims==%d", details->ndims); - log_debug(netcdf_debug_level, "len==%ld", details->len); - log_debug(netcdf_debug_level, "num_elements==%d", details->num_elements); - out_str[0]='\0'; - remaining=1023; - for (int i=0;(indims) && (remaining>0);i++) { - sprintf(tmp_str, "%lld,", details->start[i]); - strncat(out_str, tmp_str, remaining); - remaining -= strlen(tmp_str); - } - log_debug(netcdf_debug_level, "start[]==%s", out_str); - out_str[0]='\0'; - remaining=1023; - for (int i=0;(indims) && (remaining>0);i++) { - sprintf(tmp_str, "%lld,", details->count[i]); - strncat(out_str, tmp_str, remaining); - remaining -= strlen(tmp_str); - } - log_debug(netcdf_debug_level, "count[]==%s", out_str); - out_str[0]='\0'; - remaining=1023; - for (int i=0;(indims) && (remaining>0);i++) { - sprintf(tmp_str, "%lld,", details->stride[i]); - strncat(out_str, tmp_str, remaining); - remaining -= strlen(tmp_str); - } - log_debug(netcdf_debug_level, "stride[]==%s", out_str); - -// recursive_print_chunk(details, offset, index, current_dim); - log_debug(netcdf_debug_level, "+++++++++++++++++++++++++++++"); - - free(index); -} - -void print_chunk(aggregation_chunk_t *c) -{ - if (c->details == NULL) { - log_debug(netcdf_debug_level, "chunk has no details. perhaps it was aggregated into another chunk."); - return; - } - print_chunk(c->details); -} - -static void recursive_copy_chunk(aggregation_chunk_details_t *src, - aggregation_chunk_details_t *dst, - long src_offset, - long dst_offset, - int *src_index, - int *dst_index, - int current_dim) -{ - long my_src_offset=0; - long my_dst_offset=0; - - if (current_dim < src->ndims-1) { - for (int i=0;icount[current_dim];i++) { - my_src_offset = src_index[current_dim]; - my_dst_offset = dst_index[current_dim]; -// log_debug(netcdf_debug_level, "join_offset(%d) start_diff[%d](%d)", -// join_offset, current_dim, src->start[current_dim] - dst->start[current_dim]); - my_dst_offset += ((src->start[current_dim] - dst->start[current_dim]) * src->datatype_size); - for (int j=current_dim+1;jndims;j++) { - my_src_offset *= src->count[j]; - my_dst_offset *= dst->count[j]; - } - - src_index[current_dim+1]=0; - dst_index[current_dim+1]=0; - recursive_copy_chunk(src, dst, src_offset+my_src_offset, dst_offset+my_dst_offset, - src_index, dst_index, current_dim+1); - src_index[current_dim] += src->datatype_size; - dst_index[current_dim] += dst->datatype_size; - } - } else { - dst_offset += ((src->start[current_dim] - dst->start[current_dim]) * src->datatype_size); - memcpy(((char *)dst->buf) + dst_offset, - ((char *)src->buf) + src_offset, - src->count[current_dim]*src->datatype_size); - } -} - -static void copy_chunk(aggregation_chunk_details_t *src, - aggregation_chunk_details_t *dst) -{ - int *src_index=(int *)calloc(src->ndims, sizeof(int)); - int *dst_index=(int *)calloc(dst->ndims, sizeof(int)); - int src_offset=0; - long dst_offset=0; - long current_dim=0; - - memset(src_index, 0, src->ndims*sizeof(int)); - memset(dst_index, 0, dst->ndims*sizeof(int)); - recursive_copy_chunk(src, dst, src_offset, dst_offset, src_index, dst_index, current_dim); - - free(src_index); - free(dst_index); -} - -aggregation_chunk_t *aggregate_chunks(aggregation_chunk_t *c1, - aggregation_chunk_t *c2, - int join_dim) -{ - aggregation_chunk_t *out=new aggregation_chunk_t; - - log_debug(netcdf_debug_level, "entered"); - - assert(c1->details->ndims == c2->details->ndims); - assert(out != NULL); - - out->details = new aggregation_chunk_details_t; - - out->details->ncid = c1->details->ncid; - out->details->varid = c1->details->varid; - out->details->ndims = c1->details->ndims; - out->details->buf = NULL; - out->details->atype = c1->details->atype; - out->details->len = c1->details->len+c2->details->len; - out->details->datatype = c1->details->datatype; - out->details->num_elements = c1->details->num_elements+c2->details->num_elements; - out->details->datatype_size = c1->details->datatype_size; - out->details->start = (MPI_Offset *)calloc(c1->details->ndims, sizeof(MPI_Offset)); - out->details->count = (MPI_Offset *)calloc(c1->details->ndims, sizeof(MPI_Offset)); - out->details->stride = (MPI_Offset *)calloc(c1->details->ndims, sizeof(MPI_Offset)); - - memcpy(out->details->start, c1->details->start, c1->details->ndims*sizeof(MPI_Offset)); - memcpy(out->details->count, c1->details->count, c1->details->ndims*sizeof(MPI_Offset)); - out->details->count[join_dim] += c2->details->count[join_dim]; - memcpy(out->details->stride, c1->details->stride, c1->details->ndims*sizeof(MPI_Offset)); - -// recursive_aggregate_chunks(c1, c2, out); - - if (c1->component_chunks.size() > 0) { - out->component_chunks.merge(c1->component_chunks); - c1->component_chunks.clear(); - destroy_chunk(c1->details); - } else { - out->component_chunks.push_back(c1->details); - } - c1->details = NULL; - if (c2->component_chunks.size() > 0) { - out->component_chunks.merge(c2->component_chunks); - c2->component_chunks.clear(); - destroy_chunk(c2->details); - } else { - out->component_chunks.push_back(c2->details); - } - c2->details = NULL; - - assert(out != NULL); - - log_debug(netcdf_debug_level, "finished"); - - return(out); -} - -/* - * Aggregate a particular variable in the file. - * - * Aggregation rules: - * - dimension count must be equal - * - strides must be equal - * - counts on matching faces must be equal - * - - * - */ -int try_aggregation(const int ncid, const int varid) -{ - int aggregation_success=FALSE; - - file_details_t *file_details=NULL; - per_var_details_t *var_details=NULL; - aggregation_chunk_t *base_chunk=NULL; - aggregation_chunk_t *candidate_chunk=NULL; - aggregation_chunk_t *new_chunk=NULL; - chunks_iterator_t base_iter, candidate_iter; - int *start_diff; - chunk_location_count_t chunk_location_count; - int dim_with_movement=-1; - - chunks_t agg_chunks; - - int failed=FALSE; - - file_details = open_file_map[ncid]; - if (file_details == NULL) { - return(aggregation_success); - } - var_details = file_details->nc_vars[varid]; - if (var_details == NULL) { - return(aggregation_success); - } - if (var_details->chunks->size() < 2) { - log_debug(netcdf_debug_level, "returning with chunk count(%d)", var_details->chunks->size()); - return(aggregation_success); - } - log_debug(netcdf_debug_level, "chunk count(%d)", var_details->chunks->size()); - - - log_debug(netcdf_debug_level, "trying aggregation - ncid(%d) varid(%d)", ncid, varid); - - var_details->chunks->sort(compare_chunks_for_aggregation); - -// log_level old=netcdf_debug_level; -// netcdf_debug_level=LOG_ALL; -// log_debug(netcdf_debug_level, "*****************"); -// log_debug(netcdf_debug_level, "start aggregation (begin list)"); -// log_debug(netcdf_debug_level, "*****************"); -// int chunk_count; -// aggregation_chunk_t **chunks = get_chunks(ncid, varid, &chunk_count); -// for (int i=0;ichunks->begin(); - base_chunk = *base_iter; - start_diff=new int[base_chunk->details->ndims]; - for (;base_iter != var_details->chunks->end(); ++base_iter) { -// log_debug(LOG_ALL, "top: base_iter loop"); - - base_chunk = *base_iter; - - //if (base_chunk != NULL) print_chunk(base_chunk); - - // look for a chunk that can be aggregated to the base chunk - candidate_iter = base_iter; - candidate_iter++; - for (;candidate_iter != var_details->chunks->end(); ++candidate_iter) { -// log_debug(LOG_ALL, "top: candidate_iter loop"); - - candidate_chunk = *candidate_iter; - - //if (candidate_chunk != NULL) print_chunk(candidate_chunk); - - failed=FALSE; - - if (base_chunk->details->ndims != candidate_chunk->details->ndims) { - continue; - } -// if (candidate_chunk->details->start[0] != base_chunk->details->start[0]) { -// continue; -// } - for (int i=0; idetails->ndims; i++) { - if (base_chunk->details->stride[i] != candidate_chunk->details->stride[i]) { - failed=TRUE; - break; - } - - start_diff[i] = candidate_chunk->details->start[i] - base_chunk->details->start[i]; - } - if (failed) continue; - - chunk_location_count.ahead_count=0; - chunk_location_count.behind_count=0; - chunk_location_count.same_count=0; - chunk_location_count.no_match_count=0; - int agg_dims=base_chunk->details->ndims; /* the number of dimensions to aggregate */ - int first_agg_dim=0; /* first dimensions to aggregate */ - for (int i=first_agg_dim; idetails->count[i])) { - // the candidate is "behind/below" and touching the base chunk in this dimension - chunk_location_count.behind_count++; - dim_with_movement=i; - } else if ((start_diff[i] > 0) && (start_diff[i] == base_chunk->details->count[i])) { - // the candidate is "ahead of/above" and touching the base chunk in this dimension - chunk_location_count.ahead_count++; - dim_with_movement=i; - } else if (start_diff[i] == 0) { - // the candidate is "equal to" the base chunk in this dimension - chunk_location_count.same_count++; - } else { - // the candidate and the base chunk don't match in this dimension - chunk_location_count.no_match_count++; - } - } - -#ifdef DEBUG - /* - * These tests can be interesting, but are not required to get the job done. - */ - if (chunk_location_count.no_match_count > 0) { - // no matching face found. can't aggregate. - continue; - } - - if (chunk_location_count.same_count == base_chunk->ndims) { - // base and candidate have same start. bad? can't aggregate. - continue; - } - - if (chunk_location_count.ahead_count > 1) { - // movement in more than one direction - continue; - } - if (chunk_location_count.behind_count > 1) { - // movement in more than one direction - continue; - } - if ((chunk_location_count.ahead_count > 0) && - (chunk_location_count.behind_count > 0)) { - // movement in more than one direction - continue; - } - - if ((chunk_location_count.ahead_count == 0) && - (chunk_location_count.behind_count == 0)) { - // possible movement, but the chunks don't touch - continue; - } -#endif - - // check that the matching faces have the same dimensions - for (int i=0; idetails->ndims; i++) { - if ((i != dim_with_movement) && - (base_chunk->details->count[i] != candidate_chunk->details->count[i])) { - failed=TRUE; - break; - } - } - if (failed) continue; - - /* - * Do NOT uncomment these print_chunk() lines in production code. - * They are *very* slow even if the debug level is set low and - * nothing is being logged. - */ -// netcdf_debug_level=LOG_ALL; -// log_debug(netcdf_debug_level, "*****************"); -// log_debug(netcdf_debug_level, "base chunk"); -// log_debug(netcdf_debug_level, "*****************"); -// if (base_chunk != NULL) print_chunk(base_chunk); -// log_debug(netcdf_debug_level, "*****************"); -// log_debug(netcdf_debug_level, "candidate chunk"); -// log_debug(netcdf_debug_level, "*****************"); -// if (candidate_chunk != NULL) print_chunk(candidate_chunk); -// netcdf_debug_level=old; - - if ((chunk_location_count.ahead_count == 1) && - (chunk_location_count.behind_count == 0) && - (chunk_location_count.same_count == agg_dims-1)) { - // aggregation is base + candidate - new_chunk = aggregate_chunks(base_chunk, candidate_chunk, dim_with_movement); - } else if ((chunk_location_count.ahead_count == 0) && - (chunk_location_count.behind_count == 1) && - (chunk_location_count.same_count == agg_dims-1)) { - // aggregation is candidate + base - new_chunk = aggregate_chunks(candidate_chunk, base_chunk, dim_with_movement); - } else { - // chunks aren't aligned - //printf("**********\nchunks are not aligned\n**********\n"); - continue; - } - - assert(new_chunk != NULL); - - /* - * Do NOT uncomment these print_chunk() lines in production code. - * They are *very* slow even if the debug level is set low and - * nothing is being logged. - */ -// netcdf_debug_level=LOG_ALL; -// log_debug(netcdf_debug_level, "*****************"); -// log_debug(netcdf_debug_level, "new chunk"); -// log_debug(netcdf_debug_level, "*****************"); -// if (new_chunk != NULL) print_chunk(new_chunk); -// netcdf_debug_level=old; - - var_details->chunks->remove(base_chunk); - var_details->chunks->remove(candidate_chunk); - delete base_chunk; - delete candidate_chunk; - - agg_chunks.push_back(new_chunk); - - aggregation_success = TRUE; - success_this_pass = TRUE; - - break; - } - if (success_this_pass == TRUE) break; - } - chunks_iterator_t agg_iter = agg_chunks.begin(); - for (;agg_iter != agg_chunks.end();agg_iter++) { - var_details->chunks->push_back(*agg_iter); - } - agg_chunks.clear(); - - delete[] start_diff; - } - -// netcdf_debug_level=LOG_ALL; -// log_debug(netcdf_debug_level, "*****************"); -// log_debug(netcdf_debug_level, "end aggregation (begin list)"); -// log_debug(netcdf_debug_level, "*****************"); -// chunks = get_chunks(ncid, varid, &chunk_count); -// for (int i=0;ichunks->begin(); - for(;dst_iter != var_details->chunks->end();dst_iter++) { - chunk_details_iterator_t component_iter=(*dst_iter)->component_chunks.begin(); - if (((*dst_iter)->details->buf == NULL) && ((*dst_iter)->details->len > 0)) { - (*dst_iter)->details->buf = (char *)malloc((*dst_iter)->details->len); - log_debug(netcdf_debug_level, "allocated dst_iter->details->buf(%p), len(%ld)", - (*dst_iter)->details->buf, - (*dst_iter)->details->len); - } else { - log_debug(netcdf_debug_level, "did not allocate dst_iter->details->buf(%p)", (*dst_iter)->details->buf); - } - for(;component_iter != (*dst_iter)->component_chunks.end();component_iter++) { - log_debug(netcdf_debug_level, "copying component"); - copy_chunk(*component_iter, (*dst_iter)->details); - log_debug(netcdf_debug_level, "destroying component"); - destroy_chunk(*component_iter); - } - (*dst_iter)->component_chunks.clear(); - } -// netcdf_debug_level=old; - -// netcdf_debug_level=LOG_ALL; - log_debug(netcdf_debug_level, "*****************"); - log_debug(netcdf_debug_level, "chunks after aggregation"); - log_debug(netcdf_debug_level, "*****************"); - base_iter = var_details->chunks->begin(); - for (;base_iter != var_details->chunks->end(); ++base_iter) { - base_chunk = *base_iter; - if (base_chunk != NULL) - print_chunk(base_chunk); - } -// netcdf_debug_level=old; - - return(aggregation_success); -} - -/* - * Aggregate all variables in the file. - * - */ -int try_aggregation(const int ncid) -{ - int aggregation_success=FALSE; - - file_details_t *file_details=NULL; - nc_var_map_iterator_t var_iter; - per_var_details_t *var_details=NULL; - - file_details = open_file_map[ncid]; - if (file_details == NULL) { - return(aggregation_success); - } - var_iter = file_details->nc_vars.begin(); - for (; var_iter != file_details->nc_vars.end(); var_iter++) { - var_details = var_iter->second; - if (!var_details) { - continue; - } - while(try_aggregation(ncid, var_details->varid) == TRUE); - } - - aggregation_success = TRUE; - - return(aggregation_success); -} - -int aggregate_data_ready_to_write(const int ncid, const int varid) -{ -// file_details_t *details = open_file_map[ncid]; -// int chunks_needed=0; - -// if (details->num_participants > 0) { -// chunks_needed = details->num_participants; -// } else { -// chunks_needed = details->participants->size(); -// } -// -// if (details->nc_vars[varid]->chunks_received == chunks_needed) { -// return TRUE; -// } - - return FALSE; -} - -int cache_data_ready_to_write(const int ncid, const int varid) -{ -// file_details_t *details = open_file_map[ncid]; -// int chunks_needed=0; - -// if (details->num_participants > 0) { -// chunks_needed = details->num_participants; -// } else { -// chunks_needed = details->participants->size(); -// } -// -// if (details->nc_vars[varid]->chunks_received == chunks_needed) { -// return TRUE; -// } - - return FALSE; -} - -aggregation_chunk_details_t **get_chunks(const int ncid, const int varid, int *chunk_count) -{ - file_details_t *details=NULL; - per_var_details_t *var_details=NULL; - aggregation_chunk_details_t **chunks=NULL; - chunks_iterator_t iter; - - log_debug(netcdf_debug_level, "entered"); - - *chunk_count=0; - - details = open_file_map[ncid]; - if (details == NULL) { - return(NULL); - } - var_details = details->nc_vars[varid]; - if (var_details == NULL) { - return(NULL); - } - - *chunk_count = details->nc_vars[varid]->chunks->size(); - - log_debug(netcdf_debug_level, "found %d chunks to return", *chunk_count); - - if (*chunk_count == 0) { - return(NULL); - } - chunks = (aggregation_chunk_details_t **)malloc(*chunk_count*sizeof(aggregation_chunk_details_t *)); - - var_details->chunks->sort(compare_chunks_for_caching); - - iter = var_details->chunks->begin(); - for (int i=0;iter != var_details->chunks->end(); ++iter,i++) { - chunks[i] = (*iter)->details; -// print_chunk(chunks[i]); - } - - log_debug(netcdf_debug_level, "finished"); - - return(chunks); -} - -aggregation_chunk_details_t **get_chunks(const int ncid, int *chunk_count) -{ - file_details_t *details=NULL; - nc_var_map_iterator_t var_iter; - per_var_details_t *var_details=NULL; - aggregation_chunk_details_t **chunks=NULL; - chunks_iterator_t chunks_iter; - - log_debug(netcdf_debug_level, "entered"); - - *chunk_count=0; - - details = open_file_map[ncid]; - if (details == NULL) { - return(NULL); - } - var_iter = details->nc_vars.begin(); - for (; var_iter != details->nc_vars.end(); ++var_iter) { - var_details = var_iter->second; - if (!var_details) { - continue; - } - *chunk_count += var_details->chunks->size(); - } - - log_debug(netcdf_debug_level, "found %d chunks to return", *chunk_count); - - if (*chunk_count == 0) { - return(NULL); - } - chunks = (aggregation_chunk_details_t **)malloc(*chunk_count*sizeof(aggregation_chunk_details_t *)); - - int i=0; - var_iter = details->nc_vars.begin(); - for (; var_iter != details->nc_vars.end(); var_iter++) { - var_details = var_iter->second; - if (!var_details) { - continue; - } - var_details->chunks->sort(compare_chunks_for_caching); - chunks_iter = var_details->chunks->begin(); - for (;chunks_iter != var_details->chunks->end(); ++chunks_iter,i++) { - chunks[i] = (*chunks_iter)->details; -// print_chunk(chunks[i]); - } - } - - log_debug(netcdf_debug_level, "finished"); - - return(chunks); -} diff --git a/packages/trios/services/netcdf/src/aggregation.h b/packages/trios/services/netcdf/src/aggregation.h deleted file mode 100644 index 2171a1f99fca..000000000000 --- a/packages/trios/services/netcdf/src/aggregation.h +++ /dev/null @@ -1,106 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * aggregation.h - * - * Created on: Mar 9, 2009 - * Author: thkorde - */ - -#ifndef AGGREGATION_H_ -#define AGGREGATION_H_ - -#include "netcdf_args.h" - - -struct participant_t; -struct aggregation_chunk_details_t { - participant_t *p; - - int ncid; - int varid; - int ndims; - - void *buf; /* the data */ - - int atype; /* netcdf type of data in buf*/ - size_t len; /* length of buf in bytes */ - - MPI_Datatype datatype; /* MPI type of data in buf */ - size_t num_elements; /* number of datatype elements in buf (len/sizeof(datatype)) */ - int datatype_size; - - MPI_Offset *start; /* starting corner (eg. 0,0,0 is the origin of a cube) */ - MPI_Offset *count; /* num elements in each dimension (eg. 3,3,3 is a cube of size 3) */ - MPI_Offset *stride; /* sampling interval (eg. 2,2,2 is every other element in each dimension) */ -}; -typedef struct aggregation_chunk_details_t aggregation_chunk_details_t; - -int use_aggregation(const int ncid); -int use_caching(const int ncid); -int use_collective(const int ncid); -int use_independent(const int ncid); -int use_direct(const int ncid); -void add_participant_for_file(const int ncid, - const NNTI_peer_t *caller, - const write_type write_type); -void set_participant_count(const int ncid, - const NNTI_peer_t *caller, - const write_type write_type, - const int num_participants); -void remove_participant_for_file(const int ncid, const NNTI_peer_t *caller); -void add_participant_chunk(const NNTI_peer_t *caller, - aggregation_chunk_details_t *chunk); -void cleanup_aggregation_chunks(const int ncid); -void cleanup_aggregation_chunks(const int ncid, const int varid); -int try_aggregation(const int ncid); -int try_aggregation(const int ncid, const int varid); -int aggregate_data_ready_to_write(const int ncid, const int varid); -int cache_data_ready_to_write(const int ncid, const int varid); -aggregation_chunk_details_t **get_chunks(const int ncid, int *chunk_count); -aggregation_chunk_details_t **get_chunks(const int ncid, const int varid, int *chunk_count); -void print_chunk(aggregation_chunk_details_t *c); - - - -#endif /* AGGREGATION_H_ */ diff --git a/packages/trios/services/netcdf/src/cmake/Dependencies.cmake b/packages/trios/services/netcdf/src/cmake/Dependencies.cmake deleted file mode 100644 index 6112c0e4eafd..000000000000 --- a/packages/trios/services/netcdf/src/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI Pnetcdf Netcdf) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/services/netcdf/src/create_subchunks.cpp b/packages/trios/services/netcdf/src/create_subchunks.cpp deleted file mode 100644 index e928e56c2207..000000000000 --- a/packages/trios/services/netcdf/src/create_subchunks.cpp +++ /dev/null @@ -1,537 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include -#include - -#include - -#include -#include -#include -#include - -using namespace std; - -#include "create_subchunks.h" - -#include "netcdf_debug.h" -#include "netcdf_config_parser.h" - -#include -#include "Trios_timer.h" - - -extern struct netcdf_config nc_cfg; - - -/* - * if you want to be completely generic, MAX_SUBCHUNKING_DIMS should be - * equal to NC_MAX_DIMS. However, this causes a boat-load of memory to - * allocated. Then as you traverse the array, the memory must be mapped - * into RAM which is SLOW. Most codes we see use at most 4 dims. Use 6 - * here just in case. - */ -//#define MAX_SUBCHUNKING_DIMS NC_MAX_DIMS -#define MAX_SUBCHUNKING_DIMS 6 -struct start_offset_length_service { - nc_size_t start[MAX_SUBCHUNKING_DIMS]; /* coordinates of a contiguous subchunk */ - nc_size_t count[MAX_SUBCHUNKING_DIMS]; /* dimensions of a contiguous subchunk */ - int ndims; - nc_size_t offset; - nc_size_t length; - int datatype_size; - int service; -}; -typedef struct start_offset_length_service start_offset_length_service_t; - - -static void calc_offset_length(const int ndims, - const size_t *dimlens, - const int highest_contig_dim, - const int current_dim, - const nc_size_t file_offset, - nc_size_t *current_start, - start_offset_length_service_t *sols, - int *ol_index, - const nc_size_t *start, - const nc_size_t *count, - const size_t datatype_size) -{ -// log_debug(LOG_ALL, "ndims=%d; start[0,1,2,3]=%d,%d,%d,%d; count[0,1,2,3]=%d,%d,%d,%d; highest_contig_dim=%d, " -// "current_dim=%d, file_offset=%d, current_start[0,1,2,3]=%d,%d,%d,%d; ol_index=%d", -// ndims, start[0], start[1], start[2], start[3], count[0], count[1], count[2], count[3], highest_contig_dim, -// current_dim, file_offset, current_start[0], current_start[1], current_start[2], current_start[3], *ol_index); - if (current_dim < highest_contig_dim) { - current_start[current_dim]=start[current_dim]; - for (uint64_t i=0;istart = (nc_size_t *)malloc(ndims*sizeof(nc_size_t)); - c->count = (nc_size_t *)malloc(ndims*sizeof(nc_size_t)); - c->ndims = ndims; - - return(c); -} - -static consolidated_subchunks_t *consolidate_subchunk_iterval(const int first, - const int last, /* inclusive */ - const start_offset_length_service_t *sols, - const int ndims, - const size_t *dimlens, - const int highest_contig_dim, - const nc_size_t *superchunk_start, - const nc_size_t *superchunk_count) -{ - if (first > last) { - log_debug(netcdf_debug_level,"ERROR - first > last"); - return(NULL); - } - if (first == last) { - consolidated_subchunks_t *c=create_consolidated_subchunk(ndims); - memcpy(c->start, sols[first].start, ndims*sizeof(nc_size_t)); - memcpy(c->count, sols[first].count, ndims*sizeof(nc_size_t)); - nc_size_t dim_product=1; - for (int i=0;indims=sols[first].ndims; - c->nbytes=sols[first].length; - - c->offset_into_superchunk=(dim_product*sols[first].datatype_size); - - return(c); - } - for (int i=highest_contig_dim;istart, sols[first].start, ndims*sizeof(nc_size_t)); - memcpy(c->count, superchunk_count, ndims*sizeof(nc_size_t)); - for (int i=0;icount[i] = (sols[last].start[i]-sols[first].start[i])+1; - if (incomplete_dimensions[i] != -1) { - c->count[i] = incomplete_dimensions[i]; - } - } - c->ndims=sols[first].ndims; - c->nbytes=sols[first].length*(last-first+1); // assume uniform length across all sols - - nc_size_t offset=0; - for (int i=0;ioffset_into_superchunk=(offset*sols[first].datatype_size); - } else { - log_debug(netcdf_debug_level,"there are multiple(%d) incomplete dimensions -- first subchunk is sols[%d].start[", - num_incomplete_dimensions ,first); - for (int k=0;kinsert(v->end(), c); - - return(map); - } - - - - // start with a sanity check - nc_size_t lower_dim_product=1; - for (int i=highest_contig_dim;iinsert(v->end(), c); - - first_adjacent_chunk=i; - } - consolidated_subchunks_t *c = consolidate_subchunk_iterval(first_adjacent_chunk, - num_sols-1, - sols, - ndims, - dimlens, - highest_contig_dim, - superchunk_start, - superchunk_count); - - consolidated_subchunks_vector_t *v=(*map)[sols[first_adjacent_chunk].service]; - if (v == NULL) { - v=new consolidated_subchunks_vector_t; - (*map)[sols[first_adjacent_chunk].service]=v; - } - v->insert(v->end(), c); - - return(map); -} - -consolidated_subchunks_map_t *netcdf_create_subchunks(const superchunk_t *chunk, - const int *dimids, - const size_t *dimlens, - const nc_size_t bytes_per_server2) -{ - /* - * I messed around alot before I came the realization that - * unless an entire chunkwise Z plane fits on a server, you - * have to spread subchunks of your chunk across the servers. - * Guess what? That's how mpich ADIO does it. - * - * This method ignores striding. Fix it? - */ - - int rc=NC_NOERR; - - int unlimdimid=-1; - int unlimdim_index=-1; - - start_offset_length_service_t *sols=NULL; - int ol_index=0; - nc_size_t *current_start=NULL; - int highest_contig_dimension=0; - int num_contig_subchunks=0; - consolidated_subchunks_map_t *map=NULL; - consolidated_subchunks_map_iterator_t map_iter; - - nc_size_t bytes_per_server; - nc_size_t dim_product=1; - log_debug(netcdf_debug_level, "ndims(%d)", chunk->ndims); - for (int i=0;indims;i++) { - log_debug(netcdf_debug_level, "dimlens[%d](%d)", i, dimlens[i]); - dim_product *= dimlens[i]; - } - dim_product *= chunk->datatype_size; - unlimdimid=-1; - rc = nc_inq_unlimdim(chunk->ncid, &unlimdimid); /* get ID of unlimited dimension */ - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "could not get unlimdimid"); - goto cleanup; - } - unlimdim_index=-1; - if (unlimdimid != -1) { - for (int i=0;indims;i++) { - if (unlimdimid == dimids[i]) { - unlimdim_index=i; - break; - } - } - if ((unlimdim_index != 0) || (chunk->count[unlimdim_index] != 1)) { - log_warn(netcdf_debug_level, "the unlimited dimension(%d) is not dim 0 or " - "the count in the unlimited dim(%ld) is not 1. I don't know what to do.", - unlimdim_index, chunk->count[unlimdim_index]); - } else if (chunk->start[unlimdim_index] > 0) { - nc_size_t lower_dim_product=1; - for (int i=1;indims;i++) { - lower_dim_product *= dimlens[i]; - } - nc_size_t start_offset = (chunk->start[unlimdim_index]-1) * lower_dim_product * chunk->datatype_size; - dim_product -= start_offset; - } - } - bytes_per_server = dim_product/nc_cfg.num_servers; - if (dim_product%nc_cfg.num_servers > 0) { - bytes_per_server++; - } - log_debug(netcdf_debug_level, "dim_product(%lu) datatype_size(%d) num_servers(%d) dim_product/num_servers(%lu) bytes_per_server(%lu)", - dim_product, chunk->datatype_size, nc_cfg.num_servers, dim_product/nc_cfg.num_servers, bytes_per_server); - - /* First, calculate the number of contiguous subchunks there are. */ - highest_contig_dimension=0; - for (int i=chunk->ndims-1;i>=0;i--) { - if (chunk->count[i] < dimlens[i]) { - // the previous dim was the last contig dim - highest_contig_dimension = i; - break; - } - } - num_contig_subchunks=1; - for (int i=0;icount[i]; - } - /* alloc memory for the list of offset and lengths of the contiguous subchunks */ - sols=(start_offset_length_service_t *)calloc(num_contig_subchunks, sizeof(start_offset_length_service_t)); - ol_index=0; - - log_debug(netcdf_debug_level,"ndims(%ld) highest_contig_dimension(%d) num_contig_subchunks(%d)", chunk->ndims, highest_contig_dimension, num_contig_subchunks); - - trios_declare_timer(callTime); - trios_start_timer(callTime); - current_start=(nc_size_t *)calloc(chunk->ndims, sizeof(nc_size_t)); - calc_offset_length(chunk->ndims, - dimlens, - highest_contig_dimension, - 0, - 0, - current_start, - sols, - &ol_index, - chunk->start, - chunk->count, - chunk->datatype_size); - trios_stop_timer("1st calc_offset_length", callTime); -// for (int i=0;incid, chunk->varid, -// sols[i].start[0], sols[i].start[1], sols[i].start[2], sols[i].start[3], -// sols[i].count[0], sols[i].count[1], sols[i].count[2], sols[i].count[3], -// sols[i].offset, sols[i].length); -// } - - trios_start_timer(callTime); - assign_service(dim_product, bytes_per_server, sols, num_contig_subchunks); - trios_stop_timer("assign_service", callTime); - - trios_start_timer(callTime); - map=consolidate_subchunks(num_contig_subchunks, - sols, - chunk->ndims, - dimlens, - highest_contig_dimension, - chunk->start, - chunk->count); - trios_stop_timer("consolidate_subchunks", callTime); - trios_start_timer(callTime); - for (int i=0;incid, chunk->varid, - sols[i].start[0], sols[i].start[1], sols[i].start[2], sols[i].start[3], - sols[i].count[0], sols[i].count[1], sols[i].count[2], sols[i].count[3], - sols[i].offset, sols[i].length, sols[i].service); - } - trios_stop_timer("print consolidated subchunks", callTime); - - trios_start_timer(callTime); - map_iter = map->begin(); - for (;map_iter != map->end(); map_iter++) { - consolidated_subchunks_vector_iterator_t vector_iter = (*map_iter).second->begin(); - for (;vector_iter != (*map_iter).second->end(); vector_iter++) { - log_debug(netcdf_debug_level, - "ncid(%d) varid(%d) service(%02ld) super_start[%04ld,%04ld,%04ld,%04ld] start[%04ld,%04ld,%04ld,%04ld] count[%04ld,%04ld,%04ld,%04ld] offset_into_superchunk(%ld)", - chunk->ncid, chunk->varid, - (*map_iter).first, - chunk->start[0], chunk->start[1], chunk->start[2], chunk->start[3], - (*vector_iter)->start[0], (*vector_iter)->start[1], (*vector_iter)->start[2], (*vector_iter)->start[3], - (*vector_iter)->count[0], (*vector_iter)->count[1], (*vector_iter)->count[2], (*vector_iter)->count[3], - (*vector_iter)->offset_into_superchunk); - } - } - trios_stop_timer("print final subchunks", callTime); - -cleanup: - if (sols) free(sols); - if (current_start) free(current_start); - - return(map); -} diff --git a/packages/trios/services/netcdf/src/create_subchunks.h b/packages/trios/services/netcdf/src/create_subchunks.h deleted file mode 100644 index 599de473f883..000000000000 --- a/packages/trios/services/netcdf/src/create_subchunks.h +++ /dev/null @@ -1,114 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * create_subchunks.h - * - * Created on: Mar 26, 2009 - * Author: thkorde - */ - -#ifndef CREATE_SUBCHUNKS_H_ -#define CREATE_SUBCHUNKS_H_ - - -#include -#include -#include -#include - -using namespace std; - -#include "Trios_nssi_types.h" -#include "netcdf_args.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct superchunk_t { - int ncid; - int varid; - - int ndims; - - void *buf; /* the data */ - - size_t len; /* length of buf in bytes */ - - nc_type buftype; /* the netcdf type of the data in buf */ - size_t datatype_size; - - nc_size_t *start; /* starting corner (eg. 0,0,0 is the origin of a cube) */ - nc_size_t *count; /* num elements in each dimension (eg. 3,3,3 is a cube of size 3) */ - nc_size_t *stride; /* sampling interval (eg. 2,2,2 is every other element in each dimension) */ -}; -typedef struct superchunk_t superchunk_t; - -struct consolidated_subchunks { - nc_size_t *start; /* coordinates of a consolidated subchunk */ - nc_size_t *count; /* dimensions of a consolidated subchunk */ - int ndims; - int nbytes; - nc_size_t offset_into_superchunk; /* offset into the buffer that is the chunk that contains this subchunk */ -}; -typedef struct consolidated_subchunks consolidated_subchunks_t; - -typedef list consolidated_subchunks_list_t; -typedef vector consolidated_subchunks_vector_t; -typedef vector::iterator consolidated_subchunks_vector_iterator_t; - -typedef map consolidated_subchunks_map_t; -typedef map::iterator consolidated_subchunks_map_iterator_t; - - - -consolidated_subchunks_map_t *netcdf_create_subchunks(const superchunk_t *chunk, - const int *dimids, - const size_t *dimlens, - const nc_size_t bytes_per_server); - -#ifdef __cplusplus -} -#endif - -#endif /* CREATE_SUBCHUNKS_H_ */ diff --git a/packages/trios/services/netcdf/src/ncmpi_client.cpp b/packages/trios/services/netcdf/src/ncmpi_client.cpp deleted file mode 100644 index d25b10f0b3f4..000000000000 --- a/packages/trios/services/netcdf/src/ncmpi_client.cpp +++ /dev/null @@ -1,2118 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * This is sample code generated by rpcgen. - * These are only templates and you can use them - * as a guideline for developing your own functions. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include - -#include "netcdf_args.h" -#include "netcdf_debug.h" -#include "netcdf_client_private.h" - -#include "NcGroupInfo.h" -#include "NcVarInfo.h" - -#undef USE_NC_TYPE -#define USE_NC_TYPE 1 - -#include "Trios_timer.h" - -/** Association between ncid and MPI communicators */ -map comm_map; - - -/* ************************ Private functions ******************* */ - - - -/** - * Initialize the netcdf client interface. - * - * The init function relies on environment variables to know where to - * find the netcdf service. - */ -extern "C" -int ncmpi_client_init(void) -{ - int rc = NC_NOERR; - extern int netcdf_client_init(); - - rc = netcdf_client_init(); - - return rc; -} - -extern "C" -int ncmpi_fini(void) -{ - int rc = NC_NOERR; - extern int netcdf_client_fini(void); - - rc=netcdf_client_fini(); - - return rc; -} - - - -/* ************************ NETCDF LIBRARY STUBS ******************* */ - - - -/* - * - */ -extern "C" -const char *ncmpi_strerror(int ncerr) -{ - extern const char *nc_strerror(int); - return nc_strerror(ncerr); -} - - -/** - * Collective call to create a netcdf dataset. - */ -extern "C" -int ncmpi_create( - MPI_Comm comm, - const char *path, - int cmode, - MPI_Info info, - int *ncidp) -{ - int rc = NC_NOERR; - log_level debug_level = netcdf_debug_level; - int rank, np; - extern int netcdf_client_init(void); - extern int nc_create(const char *, int, int *); - - trios_declare_timer(CreateTime); - trios_start_timer(CreateTime); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &np); - - trios_declare_timer(InitTime); - trios_start_timer(InitTime); - rc = ncmpi_client_init(); - trios_stop_timer("init", InitTime); - - trios_declare_timer(BcastTime); - trios_start_timer(BcastTime); -#if defined(TRIOS_USE_TIMERS) - MPI_Barrier(comm); -#endif - trios_stop_timer("after init barrier", BcastTime); - - log_debug(debug_level, "calling nc_create(%s, %d)", path, cmode); - - log_debug(netcdf_debug_level, "Calling nc_create"); - rc = nc_create(path, cmode, ncidp); - - trios_start_timer(BcastTime); -#if defined(TRIOS_USE_TIMERS) - MPI_Barrier(comm); -#endif - trios_stop_timer("after create barrier", BcastTime); - - if (rc != NC_NOERR) { - if (rank == 0) log_error(netcdf_debug_level, "%s", ncmpi_strerror(rc)); - return rc; - } - - trios_start_timer(BcastTime); - nc_set_file_state(*ncidp); - trios_stop_timer("set file state", BcastTime); - - /* duplicate the client-side metadata for non-root processes */ - trios_start_timer(BcastTime); - if (rank != 0) { - if (group_map.find(*ncidp) == group_map.end()) { - NcGroupInfo *group = new NcGroupInfo(*ncidp, - NcFileInfo(path, cmode, 0, 0)); - group_map[*ncidp] = group; - } - else { - log_error(netcdf_debug_level, "dataset already exists for ncid=%d", *ncidp); - rc = NC_EEXIST; - } - } - trios_stop_timer("dup client-side metadata", BcastTime); - - trios_start_timer(BcastTime); - MPI_Bcast(&rc, 1, MPI_INT, 1%np, comm); - trios_stop_timer("group replicate rc Bcast", BcastTime); - - if (rc == NC_NOERR) { - comm_map[*ncidp] = comm; - } - - /* sync all the processors */ - trios_start_timer(BcastTime); - MPI_Barrier(comm); - trios_stop_timer("end barrier", BcastTime); - - trios_stop_timer("Create", CreateTime); - - return rc; -} - - - -extern "C" -int ncmpi_open( - MPI_Comm comm, - const char *path, - int mode, - MPI_Info info, - int *ncidp) -{ - int rc = NSSI_OK; - log_level debug_level = netcdf_debug_level; - char *encoded_group = NULL; - struct nc_group group; - int rank, np; - int root = 0; - XDR group_xdrs; - size_t group_size; - - /* prototypes */ - extern int nc_open(const char *, int, int *); - - trios_declare_timer(OpenTime); - trios_start_timer(OpenTime); - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &np); - - trios_declare_timer(InitTime); - trios_start_timer(InitTime); - rc = ncmpi_client_init(); - trios_stop_timer("init", InitTime); - - memset(&group, 0, sizeof(struct nc_group)); - - log_debug(debug_level, "calling nc_open(%s, %d)", path, mode); - - log_debug(netcdf_debug_level, "Calling nc_open"); - rc = nc_open(path, mode, ncidp); - - if (rc != NC_NOERR) { - log_error(debug_level, "Could not open dataset"); - goto cleanup; - } - - nc_set_file_state(*ncidp); - - if (rank == root) log_debug(debug_level, "Dataset open on root"); - - /* At this point, the root has an open netcdf dataset. The next - * step is to use XDR to serialize the nc_group, send it to all - * the client processes so they all have a copy of the dataset - * metadata. - */ - - /* extract the nc_group and calculate its size */ - if (rank == root) { - /* First, we need to convert the NcGroupInfo to struct nc_group */ - if (group_map.find(*ncidp) == group_map.end()) { - log_error(debug_level, "could not find group for dataset"); - group_size = 0; - } - else { - group_map[*ncidp]->copyTo(group); - group_size = xdr_sizeof((xdrproc_t)&xdr_nc_group, &group); - } - } - - MPI_Bcast(&group_size, sizeof(size_t), MPI_BYTE, root, comm); - - if (rank == root) log_debug(debug_level, "group_size = %d", int(group_size)); - - if (group_size <= 0) { - rc = NC_EINVAL; - if (rank == root) log_error(debug_level, "expected non-zero size for group, got %d", - (int)group_size); - goto cleanup; - } - - /* allocate space for the serialized group */ - encoded_group = (char *)calloc(group_size, sizeof(char)); - - /* serialize the nc_group */ - if (rank == root) { - - /* create a memory stream for the result */ - xdrmem_create(&group_xdrs, encoded_group, group_size, XDR_ENCODE); - - /* encode the result */ - if (! xdr_nc_group(&group_xdrs, &group)) { - rc = NSSI_EENCODE; - } - } - - MPI_Bcast(&rc, 1, MPI_INT, root, comm); - - if (rc != NSSI_OK) { - if (rank == root) - log_error(debug_level, "unable to encode nc_group"); - goto cleanup; - } - - /* broadcast encoded result */ - MPI_Bcast(encoded_group, group_size, MPI_BYTE, root, comm); - - /* decode the result on non-root nodes */ - if (rank != root) { - - /* create a memory stream for decoding */ - xdrmem_create(&group_xdrs, encoded_group, group_size, XDR_DECODE); - - /* decode the result */ - if (! xdr_nc_group(&group_xdrs, &group)) { - rc = NSSI_EDECODE; - } - } - - MPI_Bcast(&rc, 1, MPI_INT, ((root+1)%np), comm); - if (rc != NSSI_OK) { - if (rank == root) - log_error(debug_level, "unable to decode result"); - goto cleanup; - } - - /* At this point, every process should have a copy of the nc_group. - * We need to create identical dataset entries on non-root nodes. - */ - - if (rank != root) { - - /* create an entry in list of open datasets */ - if (group_map.find(*ncidp) == group_map.end()) { - NcGroupInfo *groupInfo = new NcGroupInfo(group, NcFileInfo(path, mode, 0, 0)); - group_map[*ncidp] = groupInfo; - } - else { - log_error(debug_level, "dataset already exists for ncid=%d", *ncidp); - rc = NC_EEXIST; - goto cleanup; - } - } - - /* Associate this communicator with the dataset */ - comm_map[*ncidp] = comm; - - -cleanup: - - /* cleanup data structure allocated by XDR encoding */ - xdr_free((xdrproc_t)xdr_nc_group, (char *)&group); - - if (encoded_group) free(encoded_group); - - /* sync all the processors */ - MPI_Barrier(comm); - - trios_stop_timer("Open", OpenTime); - - if (rank == root) log_debug(debug_level, "Finished opening dataset ncid=%d, rc=%d", *ncidp, rc); - return rc; -} - - -/** - * Delete a dataset. - * - * This operation isn't mentioned in the netcdf-api document, but it is in the - * include file. It is just a wrapper around the MPI_File_delete function. - */ -extern "C" -int ncmpi_delete(char *filename, MPI_Info info) -{ - return MPI_File_delete(filename, info); -} - -/* **************** DEFINE MODE FUNCTIONS **************** */ - -/** - * Collective call to add a new dimension to an open netCDF dataset. - * All processes in the communicator must call them with the same values. - * If values do not match, an error is returned by ncmpi_enddef. - * - * The correct way to implement this is to construct a log of operations, - * then execute the sequence of operations when the enddef method is called. - */ -extern "C" -int ncmpi_def_dim( - int ncid, - const char *name, - MPI_Offset len, - int *dimidp) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - extern int nc_def_dim(int, const char *, size_t, int *); - - rc = MPI_Comm_rank(comm, &rank); - - trios_declare_timer(DefDimTime); - trios_start_timer(DefDimTime); - rc = nc_def_dim(ncid, name, (size_t) len, dimidp); - trios_stop_timer("DefDim", DefDimTime); - - if (rank != 0) { - /* add the dimension to the open dataset */ - if (group_map.find(ncid) != group_map.end()) { - group_map[ncid]->def_dim(*dimidp, name, len); - } - else { - log_error(netcdf_debug_level, "dataset for ncid=%d not found", ncid); - rc = NC_EBADID; - } - } - - return rc; -} - - -/** - * Define a variable for an open netCDF dataset. - */ -extern "C" -int ncmpi_def_var( - int ncid, - const char *name, - nc_type xtype, - int ndims, - const int dimids[], - int *varidp) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; -#if USE_NC_TYPE - extern int nc_def_var(int, const char *, nc_type, int, const int *, int *); -#else - extern int nc_def_var(int, const char *, int, int, const int *, int *); -#endif - - - rc = MPI_Comm_rank(comm, &rank); - - trios_declare_timer(DefVarTime); - trios_start_timer(DefVarTime); - rc = nc_def_var(ncid, name, xtype, ndims, dimids, varidp); - trios_stop_timer("DefVar", DefVarTime); - - if (rank != 0) { - /* add the variable to the open dataset */ - if (group_map.find(ncid) != group_map.end()) { - group_map[ncid]->def_var(*varidp, name, xtype, ndims, dimids); - } - else { - log_error(netcdf_debug_level, "dataset for ncid=%d not found", ncid); - rc = NC_EBADID; - } - } - - return rc; -} - - -/* **************** ATTRIBUTE FUNCTIONS **************** */ - - - -/** - * Collective function for put_att. - */ -extern "C" -int _ncmpi_put_att_type( - int ncid, - int varid, - const char *name, - nc_type xtype, - arg_type atype, - size_t len, - const void *obj_p) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - - /* prototpye for nc_put_att_type */ -#if USE_NC_TYPE - extern int _nc_put_att_type(int, int, const char *, nc_type, arg_type, size_t, const void *); -#else - extern int _nc_put_att_type(int, int, const char *, int, arg_type, size_t, const void *); -#endif - - rc = MPI_Comm_rank(comm, &rank); - - rc = _nc_put_att_type(ncid, varid, name, xtype, atype, len, obj_p); - - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - /* Everyone else needs to store the data in the metadata cache */ - if ((rc == NC_NOERR) && (rank != 0)) { - /* add the attribute to the local metadata */ - if (group_map.find(ncid) != group_map.end()) { - NcGroupInfo *info = group_map[ncid]; - - if (varid == NC_GLOBAL) { -#if USE_NC_TYPE - info->def_att(name, xtype, len); -#else - info->def_att(name, (int)xtype, len); -#endif - } - else { - if (info->_vars.find(varid) != info->_vars.end()) { - info->_vars[varid]->def_att(name, (nc_type)xtype, len); - } - else { - log_error(netcdf_debug_level, "unable to find varid=%d", varid); - rc = NC_ENOTVAR; - goto cleanup; - } - } - } - else { - log_error(netcdf_debug_level, "unable to find ncid=%d", ncid); - rc = NC_ENOTNC; - goto cleanup; - } - } - -cleanup: - return rc; -} - -extern "C" -int ncmpi_get_att( - int ncid, - int varid, - const char *name, - void *data) -{ - extern int nc_get_att(int, int, const char *, void *); - - int rc, rank; - MPI_Comm comm = comm_map[ncid]; - nc_type xtype; - MPI_Offset len; - size_t nbytes; - log_level debug_level = netcdf_debug_level; - - MPI_Comm_rank(comm, &rank); - - rc = ncmpi_inq_att(ncid, varid, name, &xtype, &len); - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - switch (xtype) { - - case NC_BYTE: - nbytes = len; - break; - - case NC_CHAR: - nbytes = len * sizeof(char); - break; - - case NC_SHORT: - nbytes = len * sizeof(short); - break; - - case NC_INT: - nbytes = len * sizeof(int); - break; - - case NC_FLOAT: - nbytes = len * sizeof(float); - break; - - case NC_DOUBLE: - nbytes = len * sizeof(double); - break; - - default: - rc = NC_EBADTYPE; - goto cleanup; - } - - rc = nc_get_att(ncid, varid, name, data); - - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - if (rc != NC_NOERR) { - if (rank == 0) log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - MPI_Bcast(data, nbytes, MPI_BYTE, 0, comm); - -cleanup: - return rc; -} - -extern "C" -int ncmpi_get_att_text( - int ncid, - int varid, - const char *name, - char *data) -{ - int rc; - nc_type xtype; - log_level debug_level = netcdf_debug_level; - - rc = ncmpi_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (xtype != NC_CHAR) { - log_error(debug_level, "Type mismatch"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - rc = ncmpi_get_att(ncid, varid, name, data); - -cleanup: - return rc; -} - -extern "C" -int ncmpi_get_att_short( - int ncid, - int varid, - const char *name, - short *data) -{ - int rc; - nc_type xtype; - log_level debug_level = netcdf_debug_level; - - rc = ncmpi_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (xtype != NC_SHORT) { - log_error(debug_level, "Type mismatch"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - rc = ncmpi_get_att(ncid, varid, name, data); - -cleanup: - return rc; -} - -extern "C" -int ncmpi_get_att_int( - int ncid, - int varid, - const char *name, - int *data) -{ - int rc; - nc_type xtype; - log_level debug_level = netcdf_debug_level; - - rc = ncmpi_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (xtype != NC_INT) { - log_error(debug_level, "Type mismatch"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - rc = ncmpi_get_att(ncid, varid, name, data); - -cleanup: - return rc; -} - -extern "C" -int ncmpi_get_att_float( - int ncid, - int varid, - const char *name, - float*data) -{ - int rc; - nc_type xtype; - log_level debug_level = netcdf_debug_level; - - rc = ncmpi_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (xtype != NC_FLOAT) { - log_error(debug_level, "Type mismatch"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - rc = ncmpi_get_att(ncid, varid, name, data); - -cleanup: - return rc; -} - -extern "C" -int ncmpi_get_att_double( - int ncid, - int varid, - const char *name, - double *data) -{ - int rc; - nc_type xtype; - log_level debug_level = netcdf_debug_level; - - rc = ncmpi_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (xtype != NC_DOUBLE) { - log_error(debug_level, "Type mismatch"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - rc = ncmpi_get_att(ncid, varid, name, data); - -cleanup: - return rc; -} - -/** - * Add or change a variable attribute of an open netCDF dataset. - */ -extern "C" -int ncmpi_put_att ( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const void *obj_p) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_VOID, len, obj_p); -} - -extern "C" -int ncmpi_put_att_text( - int ncid, - int varid, - const char *name, - MPI_Offset len, - const char *tp) -{ - return _ncmpi_put_att_type(ncid, varid, name, NC_CHAR, NC_ARG_TEXT, len*sizeof(char), tp); -} - -extern "C" -int ncmpi_put_att_uchar( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const unsigned char *up) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_UCHAR, len*sizeof(unsigned char), up); -} - -extern "C" -int ncmpi_put_att_schar( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const signed char *cp) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_SCHAR, len*sizeof(signed char), cp); -} - -extern "C" -int ncmpi_put_att_short ( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const short *sp) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_SHORT, len*sizeof(short), sp); -} - -extern "C" -int ncmpi_put_att_int( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const int *ip) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_INT, len*sizeof(int), ip); -} - -extern "C" -int ncmpi_put_att_long( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const long *lp) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_LONG, len*sizeof(long), lp); -} - -extern "C" -int ncmpi_put_att_float( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const float *fp) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_FLOAT, len*sizeof(float), fp); -} - -extern "C" -int ncmpi_put_att_double( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const double *dp) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_DOUBLE, len*sizeof(double), dp); -} - -extern "C" -int ncmpi_put_att_ubyte( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const unsigned char *op) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_UBYTE, len*sizeof(unsigned char), op); -} - -extern "C" -int ncmpi_put_att_ushort( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const unsigned short *op) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_USHORT, len*sizeof(unsigned short), op); -} - -extern "C" -int ncmpi_put_att_uint( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const unsigned int *op) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_UINT, len*sizeof(unsigned int), op); -} - -extern "C" -int ncmpi_put_att_longlong( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const long long *op) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_LONGLONG, len*sizeof(long long), op); -} - -extern "C" -int ncmpi_put_att_ulonglong( - int ncid, - int varid, - const char *name, - nc_type xtype, - MPI_Offset len, - const unsigned long long *op) -{ - return _ncmpi_put_att_type(ncid, varid, name, xtype, NC_ARG_ULONGLONG, len*sizeof(unsigned long long), op); -} - - -/* ******************* INQUIRY FUNCTIONS ********************** */ - - -extern "C" -int ncmpi_inq( - int ncid, - int *ndimsp, - int *nvarsp, - int *nattsp, - int *unlimdimidp) -{ - extern int nc_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp); - - return nc_inq(ncid, ndimsp, nvarsp, nattsp, unlimdimidp); -} - -extern "C" -int ncmpi_inq_att( - int ncid, - int varid, - const char *name, - nc_type *xtypep, - MPI_Offset *lenp) -{ - int rc = NC_NOERR; - - rc = ncmpi_inq_attlen(ncid, varid, name, lenp); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "%s", ncmpi_strerror(rc)); - return rc; - } - - rc = ncmpi_inq_atttype(ncid, varid, name, xtypep); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "%s", ncmpi_strerror(rc)); - return rc; - } - - return rc; -} - -extern "C" -int ncmpi_inq_attlen( - int ncid, - int varid, - const char *name, - MPI_Offset *lenp) -{ - int rc = NC_NOERR; - size_t len; - - if (group_map.find(ncid) == group_map.end()) { - return NC_EBADID; - } - - /* global attribute */ - rc = group_map[ncid]->inq_attlen(varid, name, &len); - *lenp = len; - - return rc; -} - -extern "C" -int ncmpi_inq_atttype( - int ncid, - int varid, - const char *name, - nc_type *xtypep) -{ - int rc = NC_NOERR; - - if (group_map.find(ncid) == group_map.end()) { - return NC_EBADID; - } - - /* global attribute */ -#if USE_NC_TYPE - rc = group_map[ncid]->inq_atttype(varid, name, xtypep); -#else - rc = group_map[ncid]->inq_atttype(varid, name, (int *)xtypep); -#endif - - return rc; -} - -extern "C" -int ncmpi_inq_attname( - int ncid, - int varid, - int attnum, - char *name) -{ - extern int nc_inq_attname(int, int, int, char *); - - return nc_inq_attname(ncid, varid, attnum, name); -} - - -extern "C" -int ncmpi_inq_dim( - int ncid, - int dimid, - char *name, - MPI_Offset *lenp) -{ - extern int nc_inq_dim(int ncid, int dimid, char *name, size_t *lenp); - int rc=0; - size_t nc_len=0; - - rc = nc_inq_dim(ncid, dimid, name, &nc_len); - *lenp = nc_len; - return rc; -} - -extern "C" -int ncmpi_inq_dimlen( - int ncid, - int dimid, - MPI_Offset *lenp) -{ - extern int nc_inq_dimlen(int ncid, int dimid, size_t *lenp); - int rc=0; - size_t nc_len=0; - - rc = nc_inq_dimlen(ncid, dimid, &nc_len); - *lenp = nc_len; - return rc; -} - -/* The metadata should be stored on every client, so the inquiry functions should - * not require any network ops. - */ -extern "C" -int ncmpi_inq_var( - int ncid, - int varid, - char *name, - nc_type *xtypep, - int *ndimsp, - int *dimids, - int *nattsp) -{ - extern int nc_inq_var(int, int, char *, nc_type *, int *, int *, int *); - - return nc_inq_var(ncid, varid, name, xtypep, ndimsp, dimids, nattsp); -} - -extern "C" -int ncmpi_inq_varid( - int ncid, - const char *name, - int *varidp) -{ - extern int nc_inq_varid(int, const char *, int *); - - return nc_inq_varid(ncid, name, varidp); -} - -extern "C" -int ncmpi_inq_varndims( - int ncid, - int varid, - int *ndimsp) -{ - extern int nc_inq_varndims(int, int, int*); - - return nc_inq_varndims(ncid, varid, ndimsp); -} - -extern "C" -int ncmpi_inq_vartype( - int ncid, - int varid, - nc_type *xtypep) -{ - extern int nc_inq_vartype(int, int, nc_type *); - - return nc_inq_vartype(ncid, varid, xtypep); -} - -extern "C" -int ncmpi_inq_type( - int ncid, - nc_type xtype, - char *name, - size_t *sizep) -{ - extern int nc_inq_type(int, nc_type, char *, size_t *); - - return nc_inq_type(ncid, xtype, name, sizep); -} - - - -/* ********************* FLEXIBLE DATA MODE FUNCTIONS **********************/ - -/** - * Non-collective function to store an array of variables. - * - * This implementation adjusts the count array to include the size of the - * MPI datatype, then it calls the nc_put_vars function directly. - * - * The semantics of this function are slightly different than the real - * ncmpi_put_vars. This function sets default values for startp and countp - * if they are NULL. The ncmpi spec only allows the stridep array to be null. - * - */ -extern "C" -int ncmpi_put_vars( - int ncid, - int varid, - const MPI_Offset startp[], - const MPI_Offset countp[], - const MPI_Offset stridep[], - const void *buf, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - int rc = NC_NOERR; - int mpi_datatype_size; - size_t varsize; - nc_type vartype; - int ndims; - log_level debug_level = netcdf_debug_level; - - /* prototype for nc_put_vars */ - extern int nc_put_vars(int, int, const size_t *, const size_t *, const ptrdiff_t *, const void *); - - size_t *start_copy = NULL; - size_t *count_copy = NULL; - ptrdiff_t *stride_copy = NULL; - int *dimids = NULL; - - log_debug(debug_level, "Starting ncmpi_put_vars"); - - rc = ncmpi_inq_varndims(ncid, varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "Error getting number of dimensions"); - goto cleanup; - } - - if (ndims) { - /* Check to see of the MPI datatype size is the same as the nc_type (no conversion yet) */ - rc = MPI_Type_size(datatype, &mpi_datatype_size); - if (rc != MPI_SUCCESS) { - log_error(debug_level, "Error getting size of MPI_Datatype"); - goto cleanup; - } - - /* We need to get the datatype from the metadata so we know how big the buffer should be */ - rc = ncmpi_inq_vartype(ncid, varid, &vartype); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - rc = ncmpi_inq_type(ncid, vartype, NULL, &varsize); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - if (varsize != (size_t)mpi_datatype_size) { - log_error(netcdf_debug_level, "MPI_Datatype does not match nc_type (no conversion support)"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - - if (startp) { - start_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(startp, startp+ndims, start_copy); - } - - if (countp) { - count_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(countp, countp+ndims, count_copy); - } - - if (stridep) { - stride_copy = (ptrdiff_t *) calloc(ndims, sizeof(ptrdiff_t)); - copy(stridep, stridep + ndims, stride_copy); - } - - /* call the netcdf version of this function */ - rc = nc_put_vars(ncid, varid, start_copy, count_copy, stride_copy, buf); - } - - -cleanup: - if (start_copy) free(start_copy); - if (count_copy) free(count_copy); - if (stride_copy) free(stride_copy); - if (dimids) free(dimids); - return rc; -} - - -/** - * Write the entire contents of a text variable. - */ -extern "C" -int ncmpi_put_var_text(int ncid, int varid, const char *tp) -{ - log_debug(netcdf_debug_level, "Calling nc_put_var_text"); - return ncmpi_put_vars(ncid, varid, NULL, NULL, NULL, tp, -1, MPI_CHAR); -} - -/** - * Write the entire contents of a text variable. - */ -extern "C" -int ncmpi_put_var_int(int ncid, int varid, const int *ip) -{ - log_debug(netcdf_debug_level, "Calling nc_put_var_int"); - return ncmpi_put_vars(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_INT); -} - -extern "C" -int ncmpi_put_var_float(int ncid, int varid, const float *ip) -{ - log_debug(netcdf_debug_level, "Calling nc_put_var_float"); - return ncmpi_put_vars(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_put_vara( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const void *data, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - log_debug(netcdf_debug_level, "Calling nc_get_vara_float"); - return ncmpi_put_vars(ncid, varid, startp, countp, - NULL, data, bufcount, datatype); -} - -extern "C" -int ncmpi_put_vara_short( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const short *data) -{ - log_debug(netcdf_debug_level, "Calling nc_put_vara_short"); - return ncmpi_put_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_SHORT); -} - -extern "C" -int ncmpi_put_vara_int( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const int *data) -{ - log_debug(netcdf_debug_level, "Calling nc_put_vara_int"); - return ncmpi_put_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_INT); -} - -extern "C" -int ncmpi_put_vara_float( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const float *data) -{ - log_debug(netcdf_debug_level, "Calling nc_put_vara_float"); - return ncmpi_put_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_put_vara_double( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const double*data) -{ - log_debug(netcdf_debug_level, "Calling nc_put_vara_double"); - return ncmpi_put_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_DOUBLE); -} - -/* ********************* INPUT FUNCTIONS **********************/ - -/** - * Non-collective function to get an array of variables. - * - * This implementation adjusts the count array to include the size of the - * MPI datatype, then it calls the nc_put_vars function directly. - * - * The semantics of this function are slightly different than the real - * ncmpi_put_vars. This function sets default values for startp and countp - * if they are NULL. The ncmpi spec only allows the stridep array to be null. - * - */ -extern "C" -int ncmpi_get_vars( - int ncid, - int varid, - const MPI_Offset startp[], - const MPI_Offset countp[], - const MPI_Offset stridep[], - void *buf, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - int rc = NC_NOERR; - int mpi_datatype_size; - size_t varsize; - nc_type vartype; - int ndims; - log_level debug_level = netcdf_debug_level; - - /* prototype for nc_put_vars */ - extern int nc_get_vars(int, int, const size_t *, const size_t *, const ptrdiff_t *, void *); - - size_t *start_copy = NULL; - size_t *count_copy = NULL; - ptrdiff_t *stride_copy = NULL; - - rc = ncmpi_inq_varndims(ncid, varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "Error getting number of dimensions"); - goto cleanup; - } - - if (ndims) { - /* Check to see of the MPI datatype size is the same as the nc_type (no conversion yet) */ - rc = MPI_Type_size(datatype, &mpi_datatype_size); - if (rc != MPI_SUCCESS) { - log_error(debug_level, "Error getting size of MPI_Datatype"); - goto cleanup; - } - - /* We need to get the datatype from the metadata so we know how big the buffer should be */ - rc = ncmpi_inq_vartype(ncid, varid, &vartype); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - rc = ncmpi_inq_type(ncid, vartype, NULL, &varsize); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - if (varsize != (size_t)mpi_datatype_size) { - log_error(netcdf_debug_level, "MPI_Datatype does not match nc_type (no conversion support)"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - - if (startp) { - start_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(startp, startp+ndims, start_copy); - } - - if (countp) { - count_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(countp, countp+ndims, count_copy); - } - - if (stridep) { - stride_copy = (ptrdiff_t *) calloc(ndims, sizeof(ptrdiff_t)); - copy(stridep, stridep + ndims, stride_copy); - } - - /* call the netcdf version of this function */ - rc = nc_get_vars(ncid, varid, start_copy, count_copy, stride_copy, buf); - } - - -cleanup: - if (start_copy) free(start_copy); - if (count_copy) free(count_copy); - if (stride_copy) free(stride_copy); - return rc; -} - - - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int ncmpi_get_var_text(int ncid, int varid, char *tp) -{ - log_debug(netcdf_debug_level, "Calling nc_put_var_text"); - return ncmpi_get_vars(ncid, varid, NULL, NULL, NULL, tp, -1, MPI_CHAR); -} - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int ncmpi_get_var_int(int ncid, int varid, int *ip) -{ - log_debug(netcdf_debug_level, "Calling nc_get_var_int"); - return ncmpi_get_vars(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_INT); -} - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int ncmpi_get_var_float(int ncid, int varid, float *ip) -{ - log_debug(netcdf_debug_level, "Calling nc_get_var_float"); - return ncmpi_get_vars(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_get_vara( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - void *data, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - log_debug(netcdf_debug_level, "Calling nc_get_vara_float"); - return ncmpi_get_vars(ncid, varid, startp, countp, - NULL, data, bufcount, datatype); -} - -extern "C" -int ncmpi_get_vara_short( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - short*data) -{ - log_debug(netcdf_debug_level, "Calling nc_get_vara_short"); - return ncmpi_get_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_SHORT); -} - -extern "C" -int ncmpi_get_vara_int( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - int*data) -{ - log_debug(netcdf_debug_level, "Calling nc_get_vara_int"); - return ncmpi_get_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_INT); -} - -extern "C" -int ncmpi_get_vara_float( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - float *data) -{ - log_debug(netcdf_debug_level, "Calling nc_get_vara_float"); - return ncmpi_get_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_get_vara_double( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - double*data) -{ - log_debug(netcdf_debug_level, "Calling nc_get_vara_double"); - return ncmpi_get_vars(ncid, varid, startp, countp, NULL, data, -1, MPI_DOUBLE); -} - - -/* ********************* FLEXIBLE DATA MODE FUNCTIONS **********************/ - -/** - * Collective function to store an array of variables. - * - * This implementation adjusts the count array to include the size of the - * MPI datatype, then it calls the nc_put_vars function directly. - * - * The semantics of this function are slightly different than the real - * ncmpi_put_vars. This function sets default values for startp and countp - * if they are NULL. The ncmpi spec only allows the stridep array to be null. - * - */ -extern "C" -int ncmpi_put_vars_all( - int ncid, - int varid, - const MPI_Offset startp[], - const MPI_Offset countp[], - const MPI_Offset stridep[], - const void *buf, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - int rc = NC_NOERR; - int mpi_datatype_size; - size_t varsize; - nc_type vartype; - int ndims; - log_level debug_level = netcdf_debug_level; - - /* prototype for nc_put_vars */ - extern int nc_put_vars(int, int, const size_t *, const size_t *, const ptrdiff_t *, const void *); - - size_t *start_copy = NULL; - size_t *count_copy = NULL; - ptrdiff_t *stride_copy = NULL; - int *dimids = NULL; - - log_debug(debug_level, "Starting ncmpi_put_vars_all"); - - rc = ncmpi_inq_varndims(ncid, varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "Error getting number of dimensions"); - goto cleanup; - } - - if (ndims) { - /* Check to see of the MPI datatype size is the same as the nc_type (no conversion yet) */ - rc = MPI_Type_size(datatype, &mpi_datatype_size); - if (rc != MPI_SUCCESS) { - log_error(debug_level, "Error getting size of MPI_Datatype"); - goto cleanup; - } - - /* We need to get the datatype from the metadata so we know how big the buffer should be */ - rc = ncmpi_inq_vartype(ncid, varid, &vartype); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - rc = ncmpi_inq_type(ncid, vartype, NULL, &varsize); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - if (varsize != (size_t)mpi_datatype_size) { - log_error(netcdf_debug_level, "MPI_Datatype does not match nc_type (no conversion support)"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - - if (startp) { - start_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(startp, startp+ndims, start_copy); - } - - if (countp) { - count_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(countp, countp+ndims, count_copy); - } - - if (stridep) { - stride_copy = (ptrdiff_t *) calloc(ndims, sizeof(ptrdiff_t)); - copy(stridep, stridep + ndims, stride_copy); - } - - /* call the netcdf version of this function */ - rc = nc_put_vars(ncid, varid, start_copy, count_copy, stride_copy, buf); - } - - MPI_Barrier(comm_map[ncid]); - - -cleanup: - if (start_copy) free(start_copy); - if (count_copy) free(count_copy); - if (stride_copy) free(stride_copy); - if (dimids) free(dimids); - return rc; -} - - -/** - * Write the entire contents of a text variable. - */ -extern "C" -int ncmpi_put_var_text_all(int ncid, int varid, const char *tp) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, NULL, NULL, NULL, tp, -1, MPI_CHAR); -} - -/** - * Write the entire contents of a text variable. - */ -extern "C" -int ncmpi_put_var_int_all(int ncid, int varid, const int *ip) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_INT); -} - -extern "C" -int ncmpi_put_var_float_all(int ncid, int varid, const float *ip) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_put_vara_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const void *data, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_put_vars_all(ncid, varid, startp, countp, - NULL, data, bufcount, datatype); -} - -extern "C" -int ncmpi_put_vara_short_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const short *data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_SHORT); -} - -extern "C" -int ncmpi_put_vara_int_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const int *data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_INT); -} - -extern "C" -int ncmpi_put_vara_float_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const float *data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_put_vara_double_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - const double*data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_put_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_DOUBLE); -} - -/* ********************* INPUT FUNCTIONS **********************/ - -/** - * Collective function to get an array of variables. - * - * This implementation adjusts the count array to include the size of the - * MPI datatype, then it calls the nc_put_vars function directly. - * - * The semantics of this function are slightly different than the real - * ncmpi_put_vars. This function sets default values for startp and countp - * if they are NULL. The ncmpi spec only allows the stridep array to be null. - * - */ -extern "C" -int ncmpi_get_vars_all( - int ncid, - int varid, - const MPI_Offset startp[], - const MPI_Offset countp[], - const MPI_Offset stridep[], - void *buf, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - int rc = NC_NOERR; - int mpi_datatype_size; - size_t varsize; - nc_type vartype; - int ndims; - log_level debug_level = netcdf_debug_level; - - /* prototype for nc_put_vars */ - extern int nc_get_vars(int, int, const size_t *, const size_t *, const ptrdiff_t *, void *); - - size_t *start_copy = NULL; - size_t *count_copy = NULL; - ptrdiff_t *stride_copy = NULL; - - rc = ncmpi_inq_varndims(ncid, varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "Error getting number of dimensions"); - goto cleanup; - } - - if (ndims) { - /* Check to see of the MPI datatype size is the same as the nc_type (no conversion yet) */ - rc = MPI_Type_size(datatype, &mpi_datatype_size); - if (rc != MPI_SUCCESS) { - log_error(debug_level, "Error getting size of MPI_Datatype"); - goto cleanup; - } - - /* We need to get the datatype from the metadata so we know how big the buffer should be */ - rc = ncmpi_inq_vartype(ncid, varid, &vartype); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - rc = ncmpi_inq_type(ncid, vartype, NULL, &varsize); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - goto cleanup; - } - - if (varsize != (size_t)mpi_datatype_size) { - log_error(netcdf_debug_level, "MPI_Datatype does not match nc_type (no conversion support)"); - rc = NC_EIOMISMATCH; - goto cleanup; - } - - - if (startp) { - start_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(startp, startp+ndims, start_copy); - } - - if (countp) { - count_copy = (size_t *) calloc(ndims, sizeof(size_t)); - copy(countp, countp+ndims, count_copy); - } - - if (stridep) { - stride_copy = (ptrdiff_t *) calloc(ndims, sizeof(ptrdiff_t)); - copy(stridep, stridep + ndims, stride_copy); - } - - /* call the netcdf version of this function */ - rc = nc_get_vars(ncid, varid, start_copy, count_copy, stride_copy, buf); - } - - MPI_Barrier(comm_map[ncid]); - - -cleanup: - if (start_copy) free(start_copy); - if (count_copy) free(count_copy); - if (stride_copy) free(stride_copy); - return rc; -} - - - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int ncmpi_get_var_text_all(int ncid, int varid, char *tp) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_put_vars_all"); - return ncmpi_get_vars_all(ncid, varid, NULL, NULL, NULL, tp, -1, MPI_CHAR); -} - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int ncmpi_get_var_int_all(int ncid, int varid, int *ip) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_INT); -} - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int ncmpi_get_var_float_all(int ncid, int varid, float *ip) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, NULL, NULL, NULL, ip, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_get_vara_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - void *data, - MPI_Offset bufcount, - MPI_Datatype datatype) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, startp, countp, - NULL, data, bufcount, datatype); -} - -extern "C" -int ncmpi_get_vara_short_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - short*data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_SHORT); -} - -extern "C" -int ncmpi_get_vara_int_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - int*data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_INT); -} - -extern "C" -int ncmpi_get_vara_float_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - float *data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_FLOAT); -} - -extern "C" -int ncmpi_get_vara_double_all( - int ncid, - int varid, - const MPI_Offset *startp, - const MPI_Offset *countp, - double*data) -{ - log_debug(netcdf_debug_level, "Calling ncmpi_get_vars_all"); - return ncmpi_get_vars_all(ncid, varid, startp, countp, NULL, data, -1, MPI_DOUBLE); -} - - -/* *************** DEFINE MODE FUNCTIONS **************** */ - -extern "C" -int ncmpi_redef(int ncid) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - extern int nc_redef(int ncid); - - MPI_Comm_rank(comm, &rank); - - /* sync all the processors */ - /* ensure that everyone has completed their data mode ops before we switch to define mode */ - MPI_Barrier(comm); - - rc = nc_redef(ncid); - - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - nc_set_file_state(ncid); - - return rc; -} - -extern "C" -int ncmpi_enddef(int ncid) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - trios_declare_timer(callTime); - extern int nc_enddef(int ncid); - - rc = MPI_Comm_rank(comm, &rank); - - trios_start_timer(callTime); - rc = nc_enddef(ncid); - trios_stop_timer("EndDef", callTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - nc_set_file_state(ncid); - - /* sync all the processors */ - /* ensure that we have switched to data mode before everyone starts doing data mode ops */ - /* TODO: is this necessary? doesn't the MPI_Bcast ensure the no one will proceed until after rank0 calls nc_enddef? */ - trios_start_timer(callTime); - MPI_Barrier(comm); - trios_stop_timer("after enddef barrier", callTime); - - return rc; -} - - -/** - * Sync an open netCDF dataset. - */ -extern "C" -int ncmpi_sync(int ncid) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - extern int nc_sync(int ncid); - log_level debug_level = netcdf_debug_level; - - log_debug(debug_level, "Calling nc_sync"); - - MPI_Comm_rank(comm, &rank); - - /* sync all the processors */ - /* ensure that everyone has completed their outstanding ops before we sync */ - log_debug(netcdf_debug_level, "sync barrier"); - MPI_Barrier(comm); - - log_debug(netcdf_debug_level, "syncing"); - rc = nc_sync(ncid); - - log_debug(netcdf_debug_level, "sync bcast"); - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - return rc; -} - -extern "C" -int ncmpi_sync_wait(int ncid) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - log_level debug_level = netcdf_debug_level; - trios_declare_timer(SyncTime); - - log_debug(debug_level, "Calling nc_sync_wait"); - - MPI_Comm_rank(comm, &rank); - - trios_start_timer(SyncTime); - log_debug(netcdf_debug_level, "sync waiting"); - rc = nc_sync_wait(ncid); - trios_stop_timer("SyncWait nc_sync_wait()", SyncTime); - - trios_start_timer(SyncTime); - log_debug(netcdf_debug_level, "sync bcast"); - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - trios_stop_timer("SyncWait bcast()", SyncTime); - - return rc; -} - -/** - * Close an open netCDF dataset. - */ -extern "C" -int ncmpi_close(int ncid) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - extern int nc_close(int ncid); - log_level debug_level = netcdf_debug_level; - trios_declare_timer(CloseTime); - - log_debug(debug_level, "Calling nc_close"); - - MPI_Comm_rank(comm, &rank); - - /* sync all the processors */ - /* ensure that everyone has completed their outstanding ops before we close */ - trios_start_timer(CloseTime); - log_debug(netcdf_debug_level, "close barrier"); - MPI_Barrier(comm); - trios_stop_timer("Close Barrier", CloseTime); - - trios_start_timer(CloseTime); - log_debug(netcdf_debug_level, "closing"); - rc = nc_close(ncid); - trios_stop_timer("Close nc_close()", CloseTime); - - trios_start_timer(CloseTime); - log_debug(netcdf_debug_level, "close bcast"); - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - trios_stop_timer("Close bcast()", CloseTime); - - nc_set_file_state(ncid); - - return rc; -} - -extern "C" -int ncmpi_close_wait(int ncid) -{ - int rc = NC_NOERR; - int rank; - MPI_Comm comm = comm_map[ncid]; - log_level debug_level = netcdf_debug_level; - trios_declare_timer(CloseTime); - - log_debug(debug_level, "Calling nc_close_wait"); - - MPI_Comm_rank(comm, &rank); - - trios_start_timer(CloseTime); - log_debug(netcdf_debug_level, "close waiting"); - rc = nc_close_wait(ncid); - trios_stop_timer("CloseWait nc_close_wait()", CloseTime); - - trios_start_timer(CloseTime); - log_debug(netcdf_debug_level, "close bcast"); - MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - trios_stop_timer("CloseWait bcast()", CloseTime); - - trios_start_timer(CloseTime); - /* remove data structures associated with ncid */ - if (group_map.find(ncid) != group_map.end()) { - delete group_map[ncid]; - group_map.erase(ncid); - } - trios_stop_timer("group_map.erase()", CloseTime); - - trios_start_timer(CloseTime); - nc_set_file_state(ncid); - trios_stop_timer("nc_set_file_state()", CloseTime); - - return rc; -} - -extern "C" -int ncmpi_begin_indep_data(int ncid) -{ - int rc = NC_NOERR; - - rc = nc_begin_indep_data(ncid); - - nc_set_file_state(ncid); - - return rc; -} - -extern "C" -int ncmpi_end_indep_data(int ncid) -{ - int rc = NC_NOERR; - - rc = nc_end_indep_data(ncid); - - nc_set_file_state(ncid); - - return rc; -} diff --git a/packages/trios/services/netcdf/src/netcdf_args.x b/packages/trios/services/netcdf/src/netcdf_args.x deleted file mode 100644 index 9efde21166e3..000000000000 --- a/packages/trios/services/netcdf/src/netcdf_args.x +++ /dev/null @@ -1,295 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -/* -------------------------------------------------------------------------- */ -/** - * @file netcdf_args.x - * - * @brief XDR argument structures for the netcdf proxy. - * - * @author Ron Oldfield (raoldfi\@cs.sandia.gov). - * $Revision: 342 $. - * $Date: 2005-05-01 23:30:57 -0600 (Sun, 01 May 2005) $. - * - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file */ -#ifdef RPC_XDR -%#include -%#include "Trios_xdr.h" -#endif - -/** - * Operation codes for the netcdf proxy. - */ -enum netcdf_opcode { - NETCDF_NULL_OP = 10000, - NETCDF_CREATE_OP, - NETCDF_OPEN_OP, - NETCDF_DEF_DIM_OP, - NETCDF_DEF_VAR_OP, - NETCDF_GET_ATT_OP, - NETCDF_PUT_ATT_OP, - NETCDF_REDEF_OP, - NETCDF_ENDDEF_OP, - NETCDF_PUT_VARS_OP, - NETCDF_GET_VARS_OP, - NETCDF_SYNC_OP, - NETCDF_CLOSE_OP, - NETCDF_BEGIN_INDEP_OP, - NETCDF_END_INDEP_OP, - NETCDF_SET_FILL_OP -}; - - -typedef uint64_t nc_size_t; - -enum arg_type { - NC_ARG_NAT, - NC_ARG_VOID, - NC_ARG_TEXT, - NC_ARG_UCHAR, - NC_ARG_SCHAR, - NC_ARG_SHORT, - NC_ARG_INT, - NC_ARG_LONG, - NC_ARG_FLOAT, - NC_ARG_DOUBLE, - NC_ARG_UBYTE, - NC_ARG_USHORT, - NC_ARG_UINT, - NC_ARG_LONGLONG, - NC_ARG_ULONGLONG -}; - -enum extra_errcodes { - NC_ENOTSUPP = -99 -}; - -enum write_type { - WRITE_DIRECT, - WRITE_AGGREGATE_INDEPENDENT, - WRITE_AGGREGATE_COLLECTIVE, - WRITE_CACHING_INDEPENDENT, - WRITE_CACHING_COLLECTIVE -}; - -const NC_PATH_MAX = 256; - -/* ********* ARGUMENTS FOR STUB FUNCTIONS ************* */ - -/** - * Argument structure for nc_create. - */ -struct nc_create_args { - string path; - int32_t cmode; - nc_size_t initialsz; - nc_size_t chunksizehint; - enum write_type write_type; - int32_t num_participants; -}; - -/** - * Result structure for nc_create. - */ -struct nc_create_res { - int32_t ncid; - nc_size_t chunksizehint; -}; - - - -/** - * Argument structure for nc__open - */ -struct nc_open_args { - string path; - int32_t mode; - nc_size_t chunksizehint; - enum write_type write_type; - int32_t num_participants; -}; - -struct nc_dim { - int32_t dimid; - string name; - nc_size_t len; -}; - -struct nc_att { - string name; - int32_t xtype; - nc_size_t len; -}; - -struct nc_var { - int32_t varid; - string name; - int32_t xtype; - int32_t dimids; - struct nc_att atts; -}; - -/** - * A group is essentially a netcdf directory. - */ -struct nc_group { - string name; - struct nc_dim dims; - struct nc_var vars; - struct nc_att atts; - struct nc_group groups; - int32_t ncid; - int32_t parent_ncid; - int32_t unlimdimid; -}; - - -/** - * Structure for nc__open result. - * - * As an added optimization, we also return all variable - * and attribute metadata along with the chunksizehint and ncid. - */ -struct nc_open_res { - nc_size_t chunksizehint; - int32_t unlimdimid; - int32_t format; - struct nc_att gatts; - struct nc_group root_group; -}; - -/** - * Argument structure for nc_def_dim - */ -struct nc_def_dim_args { - int32_t ncid; - string name; - nc_size_t len; -}; - -/** - * Marshalled argument structure for nc_def_var - */ -struct nc_def_var_args { - int32_t ncid; - int32_t xtype; - string name; - int32_t dimids; -}; - - -/** - * Marshaled arguments for nc_put_att - */ -struct nc_put_att_args { - int32_t ncid; - int32_t varid; - string name; - int32_t xtype; - arg_type atype; - opaque data<>; -}; - -/** - * Marshalled arguments for nc_get_att - */ -struct nc_get_att_args { - int32_t ncid; - int32_t varid; - string name; -}; - - -/** - * Marshaled arguments for nc_put_var - */ -struct nc_put_vars_args { - int32_t ncid; - int32_t varid; - nc_size_t start; - nc_size_t count; - nc_size_t stride; - arg_type atype; - int32_t buftype; - nc_size_t element_count; - nc_size_t len; -}; - -typedef struct nc_put_vars_args nc_get_vars_args; - -/** - * Marshaled arguments for nc_put_var (i.e., write) - */ -struct netcdf_put_args { - int32_t unused; -}; - -/** - * Marshaled arguments for nc_set_fill - */ -struct nc_set_fill_args { - int32_t ncid; - int32_t new_fill_mode; -}; - -/** - * Result structure for nc_set_fill - */ -struct nc_set_fill_res { - int32_t old_fill_mode; -}; - -/* -program NETCDF_PROG { - version NETCDF_VERS { - data_t xfer(data_array_t) = 1; - } = 1; -} = 0x23451111; - */ diff --git a/packages/trios/services/netcdf/src/netcdf_client.cpp b/packages/trios/services/netcdf/src/netcdf_client.cpp deleted file mode 100644 index 40e9e3351c32..000000000000 --- a/packages/trios/services/netcdf/src/netcdf_client.cpp +++ /dev/null @@ -1,4126 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * This is sample code generated by rpcgen. - * These are only templates and you can use them - * as a guideline for developing your own functions. - */ - -#ifndef __STDC_CONSTANT_MACROS -#define __STDC_CONSTANT_MACROS -#endif - -#include "Trios_config.h" -#ifdef HAVE_TRIOS_PNETCDF - -#include -#include -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#include - -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - -#include "netcdf_args.h" -#include "netcdf_debug.h" /* netcdf_debug_level */ -#include "netcdf_config.h" - -#include "NcGroupInfo.h" -#include "NcVarInfo.h" - -#include - -#include "aggregation.h" - -#include "create_subchunks.h" - -#include "Trios_timer.h" - - - -/** Map of open groups */ -//map group_map; - -//static nssi_service svc; -static int default_svc; -static MPI_Comm ncmpi_collective_op_comm; -static int global_rank=-1; -static int collective_op_rank=-1; -static nssi_service *svcs; -struct netcdf_config nc_cfg; - - -size_t* NC_coord_zero; -size_t* NC_coord_one; - - -enum nc_data_mode { - NC_DEFINE_MODE, - NC_INDEP_DATA_MODE, - NC_COLL_DATA_MODE, - NC_ASYNC_CLOSED, - NC_CLOSED -}; -typedef enum nc_data_mode nc_data_mode; - -typedef list request_list_t; -typedef list::iterator request_list_iterator_t; - -struct nc_file_state { - nc_data_mode mode; - request_list_t outstanding_sync_requests; - request_list_t outstanding_close_requests; -}; -typedef struct nc_file_state nc_file_state; - -/** Association between ncid and netcdf file state */ -map file_state_map; - - -#undef BUILD_FOR_QACINA - -#if defined(BUILD_FOR_QACINA) -#define MANIPULATE_ARGS(netcdf_id) { netcdf_id--; } -#undef USE_ASYNC_CLOSE -#undef USE_ASYNC_SYNC -#else -#define MANIPULATE_ARGS(netcdf_id) { } -#define USE_ASYNC_CLOSE 1 -#define USE_ASYNC_SYNC 1 -#endif - - -/* ************************ Private functions ******************* */ - - -static int get_config_from_env(struct netcdf_config *netcdf_cfg) -{ - char *env_contact_file=getenv("NETCDF_CONTACT_INFO"); - char *env_write_mode=getenv("NSSI_WRITE_MODE"); - char *env_use_subchunking=getenv("NSSI_USE_SUBCHUNKING"); - - memset(netcdf_cfg, 0, sizeof(struct netcdf_config)); - - if ((env_write_mode==NULL) || (!strcmp(env_write_mode, "WRITE_DIRECT"))) { - netcdf_cfg->write_type = WRITE_DIRECT; - log_debug(netcdf_debug_level, "using %s", env_write_mode); - } else if (!strcmp(env_write_mode, "WRITE_AGGREGATE_INDEPENDENT")) { - netcdf_cfg->write_type = WRITE_AGGREGATE_INDEPENDENT; - log_debug(netcdf_debug_level, "using %s", env_write_mode); - } else if (!strcmp(env_write_mode, "WRITE_AGGREGATE_COLLECTIVE")) { - netcdf_cfg->write_type = WRITE_AGGREGATE_COLLECTIVE; - log_debug(netcdf_debug_level, "using %s", env_write_mode); - } else if (!strcmp(env_write_mode, "WRITE_CACHING_INDEPENDENT")) { - netcdf_cfg->write_type = WRITE_CACHING_INDEPENDENT; - log_debug(netcdf_debug_level, "using %s", env_write_mode); - } else if (!strcmp(env_write_mode, "WRITE_CACHING_COLLECTIVE")) { - netcdf_cfg->write_type = WRITE_CACHING_COLLECTIVE; - log_debug(netcdf_debug_level, "using %s", env_write_mode); - } - - sscanf(env_use_subchunking, "%lu", &netcdf_cfg->use_subchunking); - log_debug(netcdf_debug_level, "use_subchunking %lu", netcdf_cfg->use_subchunking); - - if (env_contact_file != NULL) { - int i; - FILE *f=NULL; - long fsize=0, bytes_to_read=0, bytes_read=0; - char *fbuf; - int lcount; - char *start, *end; - - struct stat sbuf; - - while (stat(env_contact_file, &sbuf)) { log_debug(netcdf_debug_level, "%s: %s", env_contact_file, strerror(errno)); usleep(500); } - - f=fopen(env_contact_file, "r"); - if (!f) { - perror(env_contact_file); - } else { - fsize = sbuf.st_size; - fbuf=(char *)calloc(fsize+1,1); - - bytes_to_read=fsize; - - while(bytes_to_read > 0) { - bytes_read+=fread(fbuf+bytes_read, 1, bytes_to_read, f); - if (bytes_read != fsize) { - log_error(netcdf_debug_level, "bytes_read(%lld) != fsize(%lld)", (int64_t)bytes_read, (int64_t)fsize); - } - bytes_to_read=fsize-bytes_read; - - log_debug(netcdf_debug_level, "fsize(%lld) bytes_read(%lld) bytes_to_read(%lld)", (int64_t)fsize, (int64_t)bytes_read, (int64_t)bytes_to_read); - } - - fclose(f); - - lcount=0; - start=end=fbuf; - do { - end=strchr(start, '\n'); - if (end!=NULL) { - lcount++; - start=end+1; - } - } while(end!=NULL); - - netcdf_cfg->num_servers=lcount; - - log_debug(netcdf_debug_level, "lcount(%lld)", (int64_t)lcount); - - netcdf_cfg->netcdf_server_urls = (char **)calloc(lcount, sizeof(char *)); - if (!netcdf_cfg->netcdf_server_urls) { - log_error(netcdf_debug_level, "could not allocate netcdf services"); - return NSSI_ENOMEM; - } - for (i=0;inetcdf_server_urls[i] = (char *)calloc(NNTI_URL_LEN, sizeof(char)); - if (!netcdf_cfg->netcdf_server_urls[i]) { - log_error(netcdf_debug_level, "could not allocate netcdf services"); - return NSSI_ENOMEM; - } - } - start=end=fbuf; - for (i=0;inetcdf_server_urls[i], start, url_len); - netcdf_cfg->netcdf_server_urls[i][url_len]='\0'; - log_debug(netcdf_debug_level, "netcdf_server_url[%d]=%s", i, netcdf_cfg->netcdf_server_urls[i]); - start=end+1; - } - } - } - } - - return(NSSI_OK); -} - - -extern "C" -void netcdf_client_fini(void) -{ - int rc=NSSI_OK; - - log_debug(netcdf_debug_level, "enter"); - - if (collective_op_rank == 0) { - rc=nssi_kill(&svcs[default_svc], 0, 1000); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - log_debug(netcdf_debug_level, "exit"); - - return; -} - - -/** - * Initialize the netcdf client interface. - * - * The init function relies on environment variables to know where to - * find the netcdf service. - */ -extern "C" -int netcdf_client_init(void) -{ - static int initialized = 0; - int rc; - - nssi_size i; - - char *log_level_str=NULL; - char *logfile_base=NULL; - char *log_file_per_client=NULL; - - char logfile[256]; - log_level debug_level; - - int is_mpi_init, np; - - if (initialized) { - return 0; - } - - trios_declare_timer(InitTime); - trios_start_timer(InitTime); - - log_debug(netcdf_debug_level, "enter"); - - MPI_Initialized(&is_mpi_init); - if (is_mpi_init == FALSE) { - MPI_Init(NULL,NULL); - } - - MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - -#if defined(BUILD_FOR_QACINA) - /* if we exit normally, we should cleanup after ourselves */ - atexit(netcdf_client_fini); -#endif - - log_level_str=getenv("NSSI_LOG_LEVEL"); - logfile_base=getenv("NSSI_LOG_FILE"); - log_file_per_client=getenv("NSSI_LOG_FILE_PER_NODE"); - - trios_declare_timer(LogInitTime); - trios_start_timer(LogInitTime); - if (logfile_base != NULL) { - if (log_file_per_client != NULL) { - sprintf(logfile, "%s.%04d", logfile_base, global_rank); - } else { - sprintf(logfile, "%s", logfile_base); - } - } - if (log_level_str != NULL) { - sscanf(log_level_str, "%d", (int*)&debug_level); - } - if (logfile_base != NULL) { - logger_init(debug_level, logfile); - } else { - logger_init(debug_level, NULL); - } - netcdf_debug_level=debug_level; - trios_stop_timer("log init", LogInitTime); - - - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - /* Register the client operations */ - NSSI_REGISTER_CLIENT_STUB(NETCDF_CREATE_OP, nc_create_args, void, nc_create_res); - NSSI_REGISTER_CLIENT_STUB(NETCDF_OPEN_OP, nc_open_args, void, nc_open_res); - NSSI_REGISTER_CLIENT_STUB(NETCDF_DEF_DIM_OP, nc_def_dim_args, void, int); - NSSI_REGISTER_CLIENT_STUB(NETCDF_DEF_VAR_OP, nc_def_var_args, void, int); - NSSI_REGISTER_CLIENT_STUB(NETCDF_GET_ATT_OP, nc_get_att_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_PUT_ATT_OP, nc_put_att_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_PUT_VARS_OP, nc_put_vars_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_GET_VARS_OP, nc_get_vars_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_REDEF_OP, int, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_ENDDEF_OP, int, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_SYNC_OP, int, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_CLOSE_OP, int, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_BEGIN_INDEP_OP, int, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_END_INDEP_OP, int, void, void); - NSSI_REGISTER_CLIENT_STUB(NETCDF_SET_FILL_OP, nc_set_fill_args, void, nc_set_fill_res); - - trios_declare_timer(ContactFileTime); - trios_start_timer(ContactFileTime); - get_config_from_env(&nc_cfg); - trios_stop_timer("contact file", ContactFileTime); - /* all ranks connect to all services */ - - if (np <= nc_cfg.num_servers) { - default_svc = global_rank; - } else { - uint32_t extra_clients = np%nc_cfg.num_servers; - if ((extra_clients) > 0) { - uint32_t num_big_groups = extra_clients; - uint32_t num_small_groups = nc_cfg.num_servers - num_big_groups; - uint32_t small_group_size = np/nc_cfg.num_servers; - uint32_t big_group_size = small_group_size+1; - int32_t max_bg_rank = (big_group_size * num_big_groups) - 1; - log_debug(debug_level, "nbg(%u) nsg(%u) sgs(%u) bgs(%u) mbgr(%d)", - num_big_groups, num_small_groups, small_group_size, big_group_size, max_bg_rank); - if (global_rank <= max_bg_rank) { - default_svc = global_rank/big_group_size; - } else { - default_svc = ((global_rank - max_bg_rank - 1)/small_group_size)+num_big_groups; - } - } else { - default_svc = global_rank/(np/nc_cfg.num_servers); - } - } - - log_debug(debug_level, "global_rank(%d) default_service(%d)", global_rank, default_svc); - - /* create a new communicator for just those clients, who share a default service. */ - MPI_Comm_split(MPI_COMM_WORLD, default_svc, global_rank, &ncmpi_collective_op_comm); - /* find my rank in the new communicator */ - MPI_Comm_rank(ncmpi_collective_op_comm, &collective_op_rank); - - log_debug(debug_level, "global_rank(%d) collective_op_rank(%d) default_service(%d)", global_rank, collective_op_rank, default_svc); - - - svcs=(nssi_service *)calloc(nc_cfg.num_servers, sizeof(nssi_service)); - /* !global_rank0 has a preferred server for data transfers. connect to preferred server. - * connect to other servers on-demand. - */ - trios_declare_timer(GetSvcTime); - trios_start_timer(GetSvcTime); - rc = nssi_get_service( - NSSI_DEFAULT_TRANSPORT, - nc_cfg.netcdf_server_urls[default_svc], - -1, - &svcs[default_svc]); - if (rc != NSSI_OK) { - log_error(debug_level, "Couldn't connect to netcdf master: %s", nssi_err_str(rc)); - return rc; - } - trios_stop_timer("get svc[default_svc]", GetSvcTime); - - trios_stop_timer("netcdf client init", InitTime); - - NC_coord_zero = (size_t*)malloc(sizeof(size_t)*NC_MAX_VAR_DIMS); - if(NC_coord_zero == NULL) abort(); - NC_coord_one = (size_t*)malloc(sizeof(size_t)*NC_MAX_VAR_DIMS); - if(NC_coord_one == NULL) abort(); - for(i=0;imode = new_mode; - - return(state); -} - -/* - * These functions replace the normal netcdf library functions. - */ -extern "C" -int _nc_create( - const char *path, - int cmode, - size_t initialsz, - size_t *chunksizehintp, - int *ncidp) -{ - int rc = NSSI_OK; - nc_create_args args; - log_level debug_level = netcdf_debug_level; - nc_create_res res; - trios_declare_timer(MulticastTime); - trios_declare_timer(WaitAnyTime); - vectorreqs; - - rc = netcdf_client_init(); - if (rc != 0) { - log_error(debug_level, "failed to initialize netcdf_client"); - goto cleanup; - } - - log_debug(netcdf_debug_level, "enter"); - - log_debug(debug_level, "Calling _nc_create"); - - memset(&args, 0, sizeof(args)); - args.path = (char *)path; - args.cmode = cmode; - args.initialsz = initialsz; - args.chunksizehint = *chunksizehintp; - args.write_type = nc_cfg.write_type; - args.num_participants = nc_cfg.num_participants; - - log_debug(debug_level, "calling _nc_create(%s, %d)", path, cmode); - - if (collective_op_rank == 0) { - trios_start_timer(MulticastTime); - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_CREATE_OP, - &args, - NULL, - 0, - &res); - trios_stop_timer("create Multicast", MulticastTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - - *ncidp = res.ncid; - - /* share the file descriptor with the rest of our group */ - MPI_Bcast(ncidp, 1, MPI_INT, 0, ncmpi_collective_op_comm); - - log_debug(debug_level, "got result rc=%d, ncid=%d", rc, *ncidp); - - if (rc == NC_NOERR) { - *chunksizehintp = res.chunksizehint; - file_state_map[*ncidp]=new_file_state(NC_DEFINE_MODE); - if (global_rank == 0) { - /* create the client-side metadata for the dataset */ - if (group_map.find(*ncidp) == group_map.end()) { - NcGroupInfo *group = new NcGroupInfo(*ncidp, - NcFileInfo(path, cmode, initialsz, *chunksizehintp)); - group_map[*ncidp] = group; - } - else { - log_error(debug_level, "dataset already exists for ncid=%d", *ncidp); - rc = NC_EEXIST; - goto cleanup; - } - } - } - -cleanup: - xdr_free((xdrproc_t)xdr_nc_create_res, (char *)&res); - - log_debug(debug_level, "Finished creating dataset ncid=%d, rc=%d", *ncidp, rc); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_create( - const char *path, - int cmode, - int *ncidp) -{ - int rc=NC_NOERR; - size_t initialsz = 0; - size_t chunksizehint = NC_SIZEHINT_DEFAULT; - - log_debug(netcdf_debug_level, "enter"); - - rc = _nc_create(path, cmode, initialsz, &chunksizehint, ncidp); - -#if defined(BUILD_FOR_QACINA) - (*ncidp)++; -#endif - - return(rc); -} - -extern "C" -int _nc_open( - const char *path, - int mode, - size_t *chunksizehintp, - int *ncidp) -{ - int rc = NSSI_OK; - nc_open_args args; - log_level debug_level = netcdf_debug_level; - nc_open_res res; - trios_declare_timer(MulticastTime); - trios_declare_timer(WaitAnyTime); - vectorreqs; - - rc = netcdf_client_init(); - if (rc != 0) { - log_error(debug_level, "failed to initialize netcdf_client"); - goto cleanup; - } - - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "Calling _nc_open"); - - memset(&res, 0, sizeof(res)); - memset(&args, 0, sizeof(args)); - args.path = (char *)path; - args.mode = mode; - args.chunksizehint = *chunksizehintp; - args.write_type = nc_cfg.write_type; - args.num_participants = nc_cfg.num_participants; - - log_debug(debug_level, "calling _nc_open(%s, %d)", path, mode); - - if (collective_op_rank == 0) { - trios_start_timer(MulticastTime); - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_OPEN_OP, - &args, - NULL, - 0, - &res); - trios_stop_timer("create Multicast", MulticastTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - *ncidp = res.root_group.ncid; - - /* share the file descriptor with the rest of our group */ - MPI_Bcast(ncidp, 1, MPI_INT, 0, ncmpi_collective_op_comm); - - log_debug(debug_level, "got result rc=%d, ncid=%d", rc, *ncidp); - - if (rc == NC_NOERR) { - *chunksizehintp = res.chunksizehint; - file_state_map[*ncidp]=new_file_state(NC_COLL_DATA_MODE); - if (global_rank == 0) { - /* create an entry in list of open datasets */ - if (group_map.find(*ncidp) == group_map.end()) { - NcGroupInfo *group = new NcGroupInfo(res.root_group, NcFileInfo(path, mode, 0, *chunksizehintp)); - group_map[*ncidp] = group; - } - else { - log_error(debug_level, "dataset already exists for ncid=%d", *ncidp); - rc = NC_EEXIST; - goto cleanup; - } - } - } - -cleanup: - xdr_free((xdrproc_t)xdr_nc_open_res, (char *)&res); - - log_debug(debug_level, "Finished opening dataset ncid=%d, rc=%d", *ncidp, rc); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_open( - const char *path, - int mode, - int *ncidp) -{ - int rc=NC_NOERR; - - size_t chunksizehint = NC_SIZEHINT_DEFAULT; - - log_debug(netcdf_debug_level, "enter"); - - rc = _nc_open(path, mode, &chunksizehint, ncidp); - -#if defined(BUILD_FOR_QACINA) - (*ncidp)++; -#endif - - return(rc); -} - -/** - * Delete a dataset. - * - * This operation isn't mentioned in the netcdf-api document, but it is in the - * include file. It is just a wrapper around the MPI_File_delete function. - */ -extern "C" -int nc_delete(const char *filename) -{ - log_debug(netcdf_debug_level, "enter"); - - return MPI_File_delete(const_cast(filename), MPI_INFO_NULL); -} - -extern "C" -int _nc_set_fill(int ncid, int fillmode, int *old_modep) -{ - *old_modep = fillmode; - int rc = NC_NOERR; - trios_declare_timer(MulticastTime); - trios_declare_timer(WaitAnyTime); - vectorreqs; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "Calling _nc_set_fill (fillmode=%d)", fillmode); - - nc_set_fill_args args; - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.new_fill_mode = fillmode; - - nc_set_fill_res res; - memset(&res, 0, sizeof(res)); - - if (collective_op_rank == 0) { - trios_start_timer(MulticastTime); - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_SET_FILL_OP, - &args, - NULL, - 0, - &res); - trios_stop_timer("_nc_set_fill Multicast", MulticastTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - *old_modep=res.old_fill_mode; - - log_debug(netcdf_debug_level, "got result from _nc_set_fill: rc=%d, fillmode=%d, old_modep=%d",rc, fillmode, *old_modep); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_set_fill(int ncid, int fillmode, int *old_modep) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_set_fill(ncid, fillmode, old_modep); -} - -extern "C" -int _nc_begin_indep_data(int ncid) -{ - int rc = NC_NOERR; - log_level debug_level = netcdf_debug_level; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(debug_level, "Calling remote _nc_begin_indep_stub"); - - MPI_Barrier(MPI_COMM_WORLD); - - if (collective_op_rank == 0) { - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_BEGIN_INDEP_OP, - &ncid, - NULL, - 0, - NULL); - - file_state_map[ncid]->mode = NC_INDEP_DATA_MODE; - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - MPI_Barrier(MPI_COMM_WORLD); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_begin_indep_data(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_begin_indep_data(ncid); -} - -extern "C" -int _nc_end_indep_data(int ncid) -{ - int rc = NC_NOERR; - log_level debug_level = netcdf_debug_level; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(debug_level, "Calling remote _nc_end_indep_stub"); - - MPI_Barrier(MPI_COMM_WORLD); - - if (collective_op_rank == 0) { - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_END_INDEP_OP, - &ncid, - NULL, - 0, - NULL); - - file_state_map[ncid]->mode = NC_COLL_DATA_MODE; - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - MPI_Barrier(MPI_COMM_WORLD); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_end_indep_data(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_end_indep_data(ncid); -} - -/** - * Add a new dimension to an open netCDF dataset. - */ -extern "C" -int _nc_def_dim( - int ncid, - const char *name, - size_t len, - int *dimidp) -{ - int rc = NC_NOERR; - int dimid; - trios_declare_timer(MulticastTime); - trios_declare_timer(WaitAnyTime); - vectorreqs; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "Calling _nc_def_dim"); - - assert(file_state_map[ncid]->mode == NC_DEFINE_MODE); - - nc_def_dim_args args; - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.name = (char *)name; - args.len = len; - - if (collective_op_rank == 0) { - trios_start_timer(MulticastTime); - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_DEF_DIM_OP, - &args, - NULL, - 0, - &dimid); - trios_stop_timer("create Multicast", MulticastTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - *dimidp=dimid; - - /* share the file descriptor with the rest of our group */ - MPI_Bcast(dimidp, 1, MPI_INT, 0, ncmpi_collective_op_comm); - - log_debug(netcdf_debug_level, "got result from _nc_def_dim: rc=%d, dimidp=%d",rc, *dimidp); - - if (global_rank == 0) { - /* add the dimension to the open dataset */ - if (group_map.find(ncid) != group_map.end()) { - group_map[ncid]->def_dim(*dimidp, name, len); - } - else { - log_error(netcdf_debug_level, "dataset for ncid=%d not found", ncid); - rc = NC_EBADID; - goto cleanup; - } - } - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_def_dim( - int ncid, - const char *name, - size_t len, - int *dimidp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_def_dim(ncid, name, len, dimidp); -} - - -/** - * Define a variable for an open netCDF dataset. - */ -extern "C" -int _nc_def_var( - int ncid, - const char *name, - nc_type xtype, - int ndims, - const int dimids[], - int *varidp) -{ - int rc = NC_NOERR; - int varid; - trios_declare_timer(MulticastTime); - trios_declare_timer(WaitAnyTime); - - vectorreqs; - - log_debug(netcdf_debug_level, "enter"); - - assert(file_state_map[ncid]->mode == NC_DEFINE_MODE); - - nc_def_var_args args; - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.name = (char *)name; - args.xtype = (int)xtype; - args.dimids.dimids_len = ndims; - args.dimids.dimids_val = (int *)&dimids[0]; - - log_debug(netcdf_debug_level, "Calling _nc_def_var(ncid=%d, name=\"%s\", xtype=%d," - " ndims=%d, dimids[0]=%d", ncid, name, xtype, ndims, args.dimids.dimids_val[0]); - - if (collective_op_rank == 0) { - trios_start_timer(MulticastTime); - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_DEF_VAR_OP, - &args, - NULL, - 0, - &varid); - trios_stop_timer("create Multicast", MulticastTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - *varidp=varid; - - /* share the file descriptor with the rest of our group */ - MPI_Bcast(varidp, 1, MPI_INT, 0, ncmpi_collective_op_comm); - - log_debug(netcdf_debug_level, "got result from _nc_def_var: rc=%d, varidp=%d",rc, *varidp); - - if (global_rank == 0) { - /* add the variable to the open dataset */ - if (group_map.find(ncid) != group_map.end()) { - group_map[ncid]->def_var(*varidp, name, xtype, ndims, dimids); - } - else { - log_error(netcdf_debug_level, "dataset for ncid=%d not found", ncid); - rc = NC_EBADID; - goto cleanup; - } - } - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_def_var( - int ncid, - const char *name, - nc_type xtype, - int ndims, - const int dimids[], - int *varidp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_def_var(ncid, name, xtype, ndims, dimids, varidp); -} - -/* ************** INQUIRY FUNCTIONS ***************** */ - -extern "C" -int _nc_inq( - int ncid, - int *ndimsp, - int *nvarsp, - int *nattsp, - int *unlimdimidp) -{ - log_debug(netcdf_debug_level, "enter"); - - if (group_map.find(ncid) == group_map.end()) { - return NC_ENOTNC; - } - else { - return group_map[ncid]->inq(ndimsp, nvarsp, nattsp, unlimdimidp); - } -} -extern "C" -int nc_inq( - int ncid, - int *ndimsp, - int *nvarsp, - int *nattsp, - int *unlimdimidp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq(ncid, ndimsp, nvarsp, nattsp, unlimdimidp); -} - -extern "C" -int _nc_inq_attlen( - int ncid, - int varid, - const char *name, - size_t *lenp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - if (group_map.find(ncid) == group_map.end()) { - return NC_ENOTNC; - } - - /* global attribute */ - rc = group_map[ncid]->inq_attlen(varid, name, lenp); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_attlen( - int ncid, - int varid, - const char *name, - size_t *lenp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_attlen(ncid, varid, name, lenp); -} - -extern "C" -int _nc_inq_atttype( - int ncid, - int varid, - const char *name, - nc_type *xtypep) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - if (group_map.find(ncid) == group_map.end()) { - return NC_ENOTNC; - } - - rc = group_map[ncid]->inq_atttype(varid, name, xtypep); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_atttype( - int ncid, - int varid, - const char *name, - nc_type *xtypep) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_atttype(ncid, varid, name, xtypep); -} - -extern "C" -int _nc_inq_att( - int ncid, - int varid, - const char *name, - nc_type *xtypep, - size_t *lenp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter (name=%s)", name); - - rc = _nc_inq_attlen(ncid, varid, name, lenp); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "%s", nc_strerror(rc)); - return rc; - } - - rc = _nc_inq_atttype(ncid, varid, name, xtypep); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "%s", nc_strerror(rc)); - return rc; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_att( - int ncid, - int varid, - const char *name, - nc_type *xtypep, - size_t *lenp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_att(ncid, varid, name, xtypep, lenp); -} - -extern "C" -int _nc_inq_attname( - int ncid, - int varid, - int attnum, - char *name) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - if (group_map.find(ncid) == group_map.end()) { - return NC_ENOTNC; - } - - rc = group_map[ncid]->inq_attname(varid, attnum, name); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_attname( - int ncid, - int varid, - int attnum, - char *name) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_attname(ncid, varid, attnum, name); -} - -extern "C" -int _nc_inq_dim( - int ncid, - int dimid, - char *name, - size_t *lenp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_dim(dimid, name, lenp); - } - else { - rc = NC_ENOTNC; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_dim( - int ncid, - int dimid, - char *name, - size_t *lenp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_dim(ncid, dimid, name, lenp); -} - -extern "C" -int _nc_inq_dimlen( - int ncid, - int dimid, - size_t *lenp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_dimlen(dimid, lenp); - } - else { - rc = NC_ENOTNC; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_dimlen( - int ncid, - int dimid, - size_t *lenp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_dimlen(ncid, dimid, lenp); -} - -extern "C" -int _nc_inq_dimid( - int ncid, - const char *name, - int *dimidp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - log_debug(netcdf_debug_level, "looking for dim named '%s'", name); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_dimid(name, dimidp); - log_debug(netcdf_debug_level, "dim named '%s' has id '%d'", name, *dimidp); - } - else { - rc = NC_EBADID; - } - - log_debug(netcdf_debug_level, "exit (rc=%d)", rc); - - return rc; -} -extern "C" -int nc_inq_dimid( - int ncid, - const char *name, - int *dimidp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_dimid(ncid, name, dimidp); -} - -extern "C" -int _nc_inq_unlimdim( - int ncid, - int *unlimdimidp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_unlimdimid(unlimdimidp); - } - else { - rc = NC_ENOTNC; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_unlimdim( - int ncid, - int *unlimdimidp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_unlimdim(ncid, unlimdimidp); -} - -extern "C" -int _nc_inq_var( - int ncid, - int varid, - char *name, - nc_type *xtypep, - int *ndimsp, - int *dimids, - int *nattsp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "Calling _nc_inq_var"); - - /* find dataset */ - if (group_map.find(ncid) == group_map.end()) { - rc = NC_ENOTNC; - goto cleanup; - } - else { - /* find variable in dataset */ - rc = group_map[ncid]->inq_var(varid, name, xtypep, ndimsp, dimids, nattsp); - } - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_var( - int ncid, - int varid, - char *name, - nc_type *xtypep, - int *ndimsp, - int *dimids, - int *nattsp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_var(ncid, varid, name, xtypep, ndimsp, dimids, nattsp); -} - -/** - * Return the name and size of an nc_type. - * - * TODO: add support for user-defined types. - */ -extern "C" -int _nc_inq_type( - int ncid, - nc_type xtype, - char *name, - size_t *sizep) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - switch (xtype) { - - case NC_BYTE: - if (name) strcpy(name, "NC_BYTE"); - *sizep = sizeof(char); - break; - - case NC_CHAR: - if (name) strcpy(name, "NC_CHAR"); - *sizep = sizeof(char); - break; - - case NC_SHORT: - if (name) strcpy(name, "NC_SHORT"); - *sizep = sizeof(short); - break; - - case NC_INT: - if (name) strcpy(name, "NC_INT"); - *sizep = sizeof(int); - break; - - case NC_FLOAT: - if (name) strcpy(name, "NC_FLOAT"); - *sizep = sizeof(float); - break; - - case NC_DOUBLE: - if (name) strcpy(name, "NC_DOUBLE"); - *sizep = sizeof(double); - break; - - default: - log_error(netcdf_debug_level, "Unsupported type=%d", (int)xtype); - rc = NC_EBADTYPE; - break; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_type( - int ncid, - nc_type xtype, - char *name, - size_t *sizep) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_type(ncid, xtype, name, sizep); -} - -extern "C" -int _nc_inq_varid( - int ncid, - const char *name, - int *varidp) -{ - int rc = NC_NOERR; - log_level debug_level = netcdf_debug_level; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_varid(name, varidp); - } - else { - rc = NC_ENOTNC; - } - - log_debug(debug_level, "found varid=%d for variable \"%s\" in ncid=%d", - *varidp, name, ncid); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_varid( - int ncid, - const char *name, - int *varidp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_varid(ncid, name, varidp); -} - -extern "C" -int _nc_inq_vardimid( - int ncid, - int varid, - int dimids[]) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_vardimid(varid, dimids); - } - else { - rc = NC_ENOTNC; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_vardimid( - int ncid, - int varid, - int dimids[]) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_vardimid(ncid, varid, dimids); -} - -extern "C" -int _nc_inq_varndims( - int ncid, - int varid, - int *ndimsp) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_varndims(varid, ndimsp); - } - else { - rc = NC_ENOTNC; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_varndims( - int ncid, - int varid, - int *ndimsp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_varndims(ncid, varid, ndimsp); -} - -extern "C" -int _nc_inq_vartype( - int ncid, - int varid, - nc_type *xtypep) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - /* find dataset */ - if (group_map.find(ncid) != group_map.end()) { - rc = group_map[ncid]->inq_vartype(varid, xtypep); - } - else { - rc = NC_ENOTNC; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_inq_vartype( - int ncid, - int varid, - nc_type *xtypep) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_inq_vartype(ncid, varid, xtypep); -} - - - -/** - * Package up an attribute and send a request to the server. - * - * WARN: This approach does not work if the client and server - * use different byte ordering. - */ -extern "C" -int _nc_put_att_type( - int ncid, - int varid, - const char *name, - nc_type xtype, - arg_type atype, - size_t len, - const void *obj_p) -{ - int rc = NC_NOERR; - nc_put_att_args args; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "Calling _nc_put_att"); - - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.varid = varid; - args.name = (char *)name; - args.xtype = (int) xtype; - args.atype = atype; - args.data.data_len = len; - args.data.data_val = (char *)obj_p; - - if (collective_op_rank == 0) { - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_PUT_ATT_OP, - &args, - NULL, - 0, - NULL); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote _nc_put_att: %s", - nssi_err_str(rc)); - goto cleanup; - } - - if (global_rank == 0) { - /* add the attribute to the local metadata */ - if (group_map.find(ncid) != group_map.end()) { - NcGroupInfo *info = group_map[ncid]; - - if (varid == NC_GLOBAL) { - info->def_att(name, xtype, len); - } - else { - if (info->_vars.find(varid) != info->_vars.end()) { - info->_vars[varid]->def_att(name, (nc_type)xtype, len); - } - else { - log_error(netcdf_debug_level, "unable to find varid=%d", varid); - rc = NC_ENOTVAR; - goto cleanup; - } - } - } - else { - log_error(netcdf_debug_level, "unable to find ncid=%d", ncid); - rc = NC_ENOTNC; - goto cleanup; - } - } - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - - -/** - * Add or change a variable attribute of an open netCDF dataset. - */ -extern "C" -int nc_put_att ( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const void *obj_p) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_VOID, len, obj_p); -} - -extern "C" -int nc_put_att_text( - int ncid, - int varid, - const char *name, - size_t len, - const char *tp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, NC_CHAR, NC_ARG_TEXT, len*sizeof(char), tp); -} - -extern "C" -int nc_put_att_uchar( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const unsigned char *up) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_UCHAR, len*sizeof(unsigned char), up); -} - -extern "C" -int nc_put_att_schar( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const signed char *cp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_SCHAR, len*sizeof(signed char), cp); -} - -extern "C" -int nc_put_att_short ( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const short *sp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_SHORT, len*sizeof(short), sp); -} - -extern "C" -int nc_put_att_int( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const int *ip) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_INT, len*sizeof(int), ip); -} - -extern "C" -int nc_put_att_long( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const long *lp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_LONG, len*sizeof(long), lp); -} - -extern "C" -int nc_put_att_float( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const float *fp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_FLOAT, len*sizeof(float), fp); -} - -extern "C" -int nc_put_att_double( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const double *dp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_DOUBLE, len*sizeof(double), dp); -} - -extern "C" -int nc_put_att_ubyte( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const unsigned char *op) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_UBYTE, len*sizeof(unsigned char), op); -} - -extern "C" -int nc_put_att_ushort( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const unsigned short *op) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_USHORT, len*sizeof(unsigned short), op); -} - -extern "C" -int nc_put_att_uint( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const unsigned int *op) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_UINT, len*sizeof(unsigned int), op); -} - -extern "C" -int nc_put_att_longlong( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const long long *op) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_LONGLONG, len*sizeof(long long), op); -} - -extern "C" -int nc_put_att_ulonglong( - int ncid, - int varid, - const char *name, - nc_type xtype, - size_t len, - const unsigned long long *op) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_att_type(ncid, varid, name, xtype, NC_ARG_ULONGLONG, len*sizeof(unsigned long long), op); -} - - -/** - * Package up an attribute and send a request to the server. - * - * WARN: This approach does not work if the client and server - * use different byte ordering. - */ -extern "C" -int _nc_get_att( - int ncid, - int varid, - const char *name, - void *data) -{ - int rc = NC_NOERR; - nc_get_att_args args; - size_t attlen; - nc_type xtype; - size_t nbytes; - size_t typesize; - - log_level debug_level = netcdf_debug_level; - - log_debug(netcdf_debug_level, "enter (name=%s)", name); - - log_debug(debug_level, "Calling _nc_get_att"); - - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.varid = varid; - args.name = const_cast(name); - - /* Need to get information about the attribute */ - rc = _nc_inq_att(ncid, varid, name, &xtype, &attlen); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not inquire about attribute: %s", nc_strerror(rc)); - goto cleanup; - } - - rc = _nc_inq_type(ncid, xtype, NULL, &typesize); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not inquire about type: %s", nc_strerror(rc)); - goto cleanup; - } - - nbytes = attlen * typesize; - - - if (collective_op_rank == 0) { - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_GET_ATT_OP, - &args, - data, - nbytes, - NULL); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote _nc_get_att: %s", - nssi_err_str(rc)); - goto cleanup; - } - -cleanup: - log_debug(netcdf_debug_level, "exit (rc=%d)", rc); - - return rc; -} - -extern "C" -int nc_get_att( - int ncid, - int varid, - const char *name, - void *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_get_att(ncid, varid, name, data); -} - -extern "C" -int nc_get_att_text( - int ncid, - int varid, - const char *name, - char *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_CHAR */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_CHAR) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_uchar( - int ncid, - int varid, - const char *name, - unsigned char *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_UBYTE */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_UBYTE) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_schar( - int ncid, - int varid, - const char *name, - signed char *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_BYTE */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_BYTE) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_short( - int ncid, - int varid, - const char *name, - short *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_SHORT */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_SHORT) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_int( - int ncid, - int varid, - const char *name, - int *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_INT */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_INT) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_long( - int ncid, - int varid, - const char *name, - long *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_INT */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_INT) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_float( - int ncid, - int varid, - const char *name, - float *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_FLOAT */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_FLOAT) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_double( - int ncid, - int varid, - const char *name, - double *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_DOUBLE */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_DOUBLE) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_ubyte( - int ncid, - int varid, - const char *name, - unsigned char *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_UBYTE */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_UBYTE) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_ushort( - int ncid, - int varid, - const char *name, - unsigned short *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_USHORT */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_USHORT) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_uint( - int ncid, - int varid, - const char *name, - unsigned int *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_UINT */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_UINT) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_longlong( - int ncid, - int varid, - const char *name, - long long *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_INT64 */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_INT64) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_att_ulonglong( - int ncid, - int varid, - const char *name, - unsigned long long *data) -{ - int rc; - nc_type xtype; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* make sure the type is NC_UINT64 */ - rc = _nc_inq_atttype(ncid, varid, name, &xtype); - if (rc != NC_NOERR) { - return rc; - } - - if (xtype != NC_UINT64) { - return NC_EBADTYPE; - } - - rc = _nc_get_att(ncid, varid, name, data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -/* ********************* OUTPUT FUNCTIONS **********************/ - -/** - * The unlimited dimension starts out with length 0. As records are written, - * the length increases. We update the length here. - */ -static int update_unlimdim_dimlen(const int ncid, const size_t dimlen) -{ - int rc = NC_NOERR; - - if (group_map.find(ncid) != group_map.end()) { - group_map[ncid]->update_unlimdim_dimlen(dimlen); - } - else { - log_error(netcdf_debug_level, "dataset for ncid=%d not found", ncid); - rc = NC_EBADID; - } - - return(rc); -} - - -/** - * Write a subsampled array of values. - * - * This function is sufficient for all other netcdf write functions. - */ -extern "C" -int _nc_put_vars( - int ncid, - int varid, - nc_type buftype, - size_t buftypesize, /* the size of 'data' based type */ - const size_t *startp, - const size_t *countp, - const ptrdiff_t *stridep, - const void *data) -{ - int rc, i; - log_level debug_level = netcdf_debug_level; - - /* need copy in case word sizes don't match */ - nc_size_t *start_copy = NULL; - nc_size_t *count_copy = NULL; - nc_size_t *stride_copy = NULL; - int *dimids = NULL; - size_t *dimlens = NULL; - - int unlimdimid=-1; - - - NcGroupInfo *group = group_map[ncid]; - size_t nbytes = 0; - - nc_put_vars_args args; - int ndims; - - trios_declare_timer(callTime); - trios_start_timer(callTime); - - log_debug(netcdf_debug_level, "enter"); - - /* calculate the size of the buffer. What happens on overruns? Segfault? */ - if (group_map.find(ncid) == group_map.end()) { - log_error(debug_level, "Could not find group for ncid=%d", ncid); - rc = NC_EINVAL; - goto cleanup; - } - - rc = group->inq_varndims(varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "unable to get ndims"); - goto cleanup; - } - - - dimids = (int *) calloc(ndims, sizeof(int)); - dimlens = (size_t *) calloc(ndims, sizeof(size_t)); - - /* get the diminsion IDs for this variable */ - rc = _nc_inq_vardimid(ncid, varid, dimids); - if (rc != NC_NOERR) { - log_error(debug_level, "could not get dimids"); - goto cleanup; - } - - rc = _nc_inq_unlimdim(ncid, &unlimdimid); /* get ID of unlimited dimension */ - if (rc != NC_NOERR) { - log_error(debug_level, "could not get unlimdimid"); - goto cleanup; - } - for (i = 0; i(data); - chunk.len=nbytes; - chunk.buftype=buftype; - chunk.datatype_size=buftypesize; - chunk.start=start_copy; - chunk.count=count_copy; - chunk.stride=stride_copy; - - trios_declare_timer(subchunkingTime); - trios_start_timer(subchunkingTime); - consolidated_subchunks_map_t *map=netcdf_create_subchunks(&chunk, dimids, dimlens, nc_cfg.bytes_per_server); - trios_stop_timer("subchunking", subchunkingTime); - - trios_declare_timer(totalPutTime); - trios_start_timer(totalPutTime); - consolidated_subchunks_map_iterator_t map_iter = map->begin(); - for (;map_iter != map->end(); map_iter++) { - consolidated_subchunks_vector_iterator_t vector_iter = (*map_iter).second->begin(); - char outstr[1024]; - for (;vector_iter != (*map_iter).second->end(); vector_iter++) { - sprintf(outstr, "service(%02ld) super_start[", (*map_iter).first); - char tmpstr[256]; - for (i=0;istart[i]); - strcat(outstr, tmpstr); - } - strcat(outstr, "] count["); - for (i=0;icount[i]); - strcat(outstr, tmpstr); - } - sprintf(tmpstr, "] offset_into_superchunk(%ld)", (*vector_iter)->offset_into_superchunk); - strcat(outstr, tmpstr); - log_debug(debug_level, "%s", outstr); - -// log_debug(debug_level, -// "service(%02ld) super_start[%04ld,%04ld,%04ld,%04ld] start[%04ld,%04ld,%04ld,%04ld] count[%04ld,%04ld,%04ld,%04ld] offset_into_superchunk(%ld)", -// (*map_iter).first, -// chunk.start[0], chunk.start[1], chunk.start[2], chunk.start[3], -// (*vector_iter)->start[0], (*vector_iter)->start[1], (*vector_iter)->start[2], (*vector_iter)->start[3], -// (*vector_iter)->count[0], (*vector_iter)->count[1], (*vector_iter)->count[2], (*vector_iter)->count[3], -// (*vector_iter)->offset_into_superchunk); - - - /* initialize remote arguments */ - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.varid = varid; - args.atype = NC_ARG_VOID; /* unused */ - args.start.start_len = (start_copy != NULL)? ndims: 0; - args.start.start_val = (*vector_iter)->start; - args.count.count_len = (count_copy != NULL)? ndims: 0; - args.count.count_val = (*vector_iter)->count; - args.stride.stride_len = (stride_copy != NULL)? ndims: 0; - args.stride.stride_val = stride_copy; - args.buftype = buftype; - args.element_count = count; - args.len = (*vector_iter)->nbytes; - - log_debug(debug_level, "sending start_len=%d, count_len=%d, stride_len=%d", - args.start.start_len, args.count.count_len, args.stride.stride_len); - - int svc_index=(*map_iter).first; - log_debug(debug_level, "svc_index(%d)", svc_index); - - trios_declare_timer(putTime); - trios_start_timer(putTime); - if (((nc_cfg.write_type != WRITE_AGGREGATE_COLLECTIVE) && - (nc_cfg.write_type != WRITE_CACHING_COLLECTIVE)) && - (file_state_map[ncid]->mode != NC_INDEP_DATA_MODE)) { - _nc_begin_indep_data(ncid); - /* Send request to remote service */ - rc = nssi_call_rpc_sync(&svcs[svc_index], NETCDF_PUT_VARS_OP, &args, - ((char *)data)+(*vector_iter)->offset_into_superchunk, (*vector_iter)->nbytes, NULL); - if (rc != NSSI_OK) { - log_error(debug_level, "unable to call remote _nc_put_vars"); - goto cleanup; - } - _nc_end_indep_data(ncid); - } else { - /* Send request to remote service */ - rc = nssi_call_rpc_sync(&svcs[svc_index], NETCDF_PUT_VARS_OP, &args, - ((char *)data)+(*vector_iter)->offset_into_superchunk, (*vector_iter)->nbytes, NULL); - if (rc != NSSI_OK) { - log_error(debug_level, "unable to call remote _nc_put_vars"); - goto cleanup; - } - } - trios_stop_timer("subchunk put", putTime); - } - } - trios_stop_timer("total subchunk put", totalPutTime); - } else { -// log_debug(LOG_ALL, "NOT subchunking"); - /* initialize remote arguments */ - memset(&args, 0, sizeof(args)); - args.ncid = ncid; - args.varid = varid; - args.atype = NC_ARG_VOID; /* unused */ - args.start.start_len = (start_copy != NULL)? ndims: 0; - args.start.start_val = start_copy; - args.count.count_len = (count_copy != NULL)? ndims: 0; - args.count.count_val = count_copy; - args.stride.stride_len = (stride_copy != NULL)? ndims: 0; - args.stride.stride_val = stride_copy; - args.buftype = buftype; - args.element_count = count; - args.len = nbytes; - - log_debug(debug_level, "sending start_len=%d, count_len=%d, stride_len=%d", - args.start.start_len, args.count.count_len, args.stride.stride_len); - - int svc_index=default_svc; - log_debug(debug_level, "svc_index(%d)", svc_index); - -// log_debug(LOG_ALL, -// "service(%02ld) ncid(%d) varid(%d) ndims(%d) start[%04ld,%04ld,%04ld,%04ld] count[%04ld,%04ld,%04ld,%04ld]", -// svc_index, ncid, varid, ndims, -// start_copy[0], start_copy[1], start_copy[2], start_copy[3], -// count_copy[0], count_copy[1], count_copy[2], count_copy[3]); - - - trios_declare_timer(putTime); - trios_start_timer(putTime); - if (((nc_cfg.write_type != WRITE_AGGREGATE_COLLECTIVE) && - (nc_cfg.write_type != WRITE_CACHING_COLLECTIVE)) && - (file_state_map[ncid]->mode != NC_INDEP_DATA_MODE)) { - _nc_begin_indep_data(ncid); - /* Send request to remote service */ - rc = nssi_call_rpc_sync(&svcs[svc_index], NETCDF_PUT_VARS_OP, &args, (char *)data, nbytes, NULL); - if (rc != NSSI_OK) { - log_error(debug_level, "unable to call remote _nc_put_vars"); - goto cleanup; - } - _nc_end_indep_data(ncid); - } else { - /* Send request to remote service */ - rc = nssi_call_rpc_sync(&svcs[svc_index], NETCDF_PUT_VARS_OP, &args, (char *)data, nbytes, NULL); - if (rc != NSSI_OK) { - log_error(debug_level, "unable to call remote _nc_put_vars"); - goto cleanup; - } - } - trios_stop_timer("put", putTime); - } - - if (unlimdimid != -1) { - for (i=0;iinq_varndims(varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "unable to get ndims"); - goto cleanup; - } - - - dimids = (int *) calloc(ndims, sizeof(int)); - dimlens = (size_t *) calloc(ndims, sizeof(size_t)); - - /* get the diminsion IDs for this variable */ - rc = _nc_inq_vardimid(ncid, varid, dimids); - if (rc != NC_NOERR) { - log_error(debug_level, "could not get dimids"); - goto cleanup; - } - - /* set count[i] to dimlen[i] */ - for (i = 0; imode != NC_INDEP_DATA_MODE) { - _nc_begin_indep_data(ncid); - /* Send request to remote service */ - rc = nssi_call_rpc_sync(&svcs[svc_index], NETCDF_GET_VARS_OP, &args, (char *)data, nbytes, NULL); - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote _nc_get_vars"); - goto cleanup; - } - _nc_end_indep_data(ncid); - } else { - /* Send request to remote service */ - rc = nssi_call_rpc_sync(&svcs[svc_index], NETCDF_GET_VARS_OP, &args, (char *)data, nbytes, NULL); - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote _nc_get_vars"); - goto cleanup; - } - } - - log_debug(debug_level, "first two vals = %d, %d, ...", - ((int *)data)[0], ((int *)data)[1]); - } - -cleanup: - if (start_copy) free(start_copy); - if (count_copy) free(count_copy); - if (stride_copy) free(stride_copy); - if (dimids) free(dimids); - - log_debug(debug_level, "Exiting get_vars: rc=%d", rc); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_vars( - int ncid, int varid, - const size_t *startp, - const size_t *countp, - const ptrdiff_t *stridep, - void *data) -{ - log_debug(netcdf_debug_level, "enter"); - - return _nc_get_vars(ncid, varid, startp, countp, stridep, data); -} - -/** - * Write a subsampled array of float values. - */ -extern "C" -int nc_get_vars_float( - int ncid, int varid, - const size_t *startp, - const size_t *countp, - const ptrdiff_t *stridep, - float *data) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vars_float"); - rc = _nc_get_vars(ncid, varid, startp, countp, stridep, (char *)data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -/** - * Write a subsampled array of double values. - */ -extern "C" -int nc_get_vars_double( - int ncid, int varid, - const size_t *startp, - const size_t *countp, - const ptrdiff_t *stridep, - double *data) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vars_double"); - rc = _nc_get_vars(ncid, varid, startp, countp, stridep, (char *)data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -extern "C" -int nc_get_var( - int ncid, - int varid, - void *data) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_var"); - rc = _nc_get_vars(ncid, varid, NULL, NULL, NULL, (char *)data); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -/** - * Get the entire contents of a text variable. - */ -extern "C" -int nc_get_var_text(int ncid, int varid, char *tp) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_var_text"); - return _nc_get_vars(ncid, varid, NULL, NULL, NULL, tp); -} - -/** - * Get the entire contents of an int variable. - */ -extern "C" -int nc_get_var_int(int ncid, int varid, int *ip) -{ - int rc=0; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_var_int(ncid=%d)", ncid); - rc = _nc_get_vars(ncid, varid, NULL, NULL, NULL, ip); - - log_debug(netcdf_debug_level, "ip[0]=%d", ip[0]); - - return rc; -} - -/** - * Get the entire contents of a float variable. - */ -extern "C" -int nc_get_var_float(int ncid, int varid, float *fp) -{ - int rc=0; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_var_float(ncid=%d)", ncid); - rc = _nc_get_vars(ncid, varid, NULL, NULL, NULL, fp); - - log_debug(netcdf_debug_level, "fp[0]=%f", fp[0]); - - return rc; -} - -/** - * Get the entire contents of a double variable. - */ -extern "C" -int nc_get_var_double(int ncid, int varid, double *dp) -{ - int rc=0; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_var_double(ncid=%d)", ncid); - rc = _nc_get_vars(ncid, varid, NULL, NULL, NULL, dp); - - log_debug(netcdf_debug_level, "dp[0]=%f", dp[0]); - - return rc; -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - void *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara_text( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - char *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara_text(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara_int( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - int *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara_int(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara_long( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - long *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara_long(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara_longlong( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - long long *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara_longlong(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara_float( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - float *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara_float(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - -/** - * Read an array of values. - */ -extern "C" -int nc_get_vara_double( - int ncid, - int varid, - const size_t startp[], - const size_t countp[], - double *data) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - log_debug(netcdf_debug_level, "Calling _nc_get_vara_double(ncid=%d)", ncid); - return _nc_get_vars(ncid, varid, startp, countp, NULL, (char *)data); -} - - -extern "C" -int nc_put_var1( - int ncid, - int varid, - const size_t *coord, - const void* value, - nc_type memtype) -{ - int rc = NC_NOERR; - nc_type vartype; - size_t varsize; - - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - /* Get the datatype information so we know the size of the buffer */ - /* This assumes that the 'data' has the same type as the variable varid */ - rc = _nc_inq_vartype(ncid, varid, &vartype); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - return rc; - } - rc = _nc_inq_type(ncid, vartype, NULL, &varsize); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Unable to get variable type"); - return rc; - } - - return _nc_put_vars(ncid, varid, vartype, varsize, coord, NC_coord_one, NULL, value); -} - -extern "C" -int nc_put_var1_text( - int ncid, - int varid, - const size_t *coord, - const char *value) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_vars(ncid, varid, NC_CHAR, sizeof(char), coord, NC_coord_one, NULL, value); -} - -extern "C" -int nc_put_var1_int( - int ncid, - int varid, - const size_t *coord, - const int *value) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_vars(ncid, varid, NC_INT, sizeof(int), coord, NC_coord_one, NULL, value); -} - -extern "C" -int nc_put_var1_float( - int ncid, - int varid, - const size_t *coord, - const float *value) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_vars(ncid, varid, NC_FLOAT, sizeof(float), coord, NC_coord_one, NULL, value); -} - -extern "C" -int nc_put_var1_double( - int ncid, - int varid, - const size_t *coord, - const double *value) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_put_vars(ncid, varid, NC_DOUBLE, sizeof(double), coord, NC_coord_one, NULL, value); -} - - -extern "C" -int nc_get_var1( - int ncid, - int varid, - const size_t *coord, - void* value, - nc_type memtype) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_get_vars(ncid, varid, coord, NC_coord_one, NULL, value); -} - -extern "C" -int nc_get_var1_int( - int ncid, - int varid, - const size_t *coord, - int *value) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_get_vars(ncid, varid, coord, NC_coord_one, NULL, value); -} - - -extern "C" -int _nc_redef(int ncid) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "Calling _nc_redef"); - - if (collective_op_rank == 0) { - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_REDEF_OP, - &ncid, - NULL, - 0, - NULL); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote _nc_redef: %s", - nssi_err_str(rc)); - goto cleanup; - } - file_state_map[ncid]->mode = NC_DEFINE_MODE; - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_redef(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_redef(ncid); -} - -extern "C" -int _nc_enddef(int ncid) -{ - int rc = NC_NOERR; - trios_declare_timer(MulticastTime); - trios_declare_timer(WaitAnyTime); - vectorreqs; - - log_debug(netcdf_debug_level, "enter"); - - if (collective_op_rank == 0) { - trios_start_timer(MulticastTime); - /* call the remote method */ - rc = nssi_call_rpc_sync(&svcs[default_svc], - NETCDF_ENDDEF_OP, - &ncid, - NULL, - 0, - NULL); - - trios_stop_timer("enddef Multicast", MulticastTime); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - - file_state_map[ncid]->mode = NC_COLL_DATA_MODE; - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_enddef(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_enddef(ncid); -} - - -/** - * Sync an open netCDF dataset. - */ -extern "C" -int _nc_sync_wait(int ncid) -{ - int rc=NSSI_OK; - nssi_request *req=NULL; - request_list_iterator_t iter; - - log_debug(netcdf_debug_level, "enter"); - - iter = file_state_map[ncid]->outstanding_sync_requests.begin(); - for (;iter != file_state_map[ncid]->outstanding_sync_requests.end();iter++) { - req = *iter; - if (req != NULL) { - nssi_wait(req, &rc); - free(req); - } - } - file_state_map[ncid]->outstanding_sync_requests.clear(); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_sync_wait(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_sync_wait(ncid); -} - -/** - * Sync an open netCDF dataset. - */ -extern "C" -int _nc_sync(int ncid) -{ - int rc=NSSI_OK; - nssi_request *req=NULL; - - log_debug(netcdf_debug_level, "enter"); - - if (collective_op_rank == 0) { - req=(nssi_request *)calloc(1, sizeof(nssi_request)); - /* call the remote method */ - rc = nssi_call_rpc(&svcs[default_svc], - NETCDF_SYNC_OP, - &ncid, - NULL, - 0, - NULL, - req); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote nc_sync: %s", - nssi_err_str(rc)); - goto cleanup; - } - - file_state_map[ncid]->outstanding_sync_requests.push_back(req); - -#if !defined(USE_ASYNC_SYNC) - rc=_nc_sync_wait(ncid); -#endif - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_sync(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_sync(ncid); -} - -/** - * Close an open netCDF dataset. - */ -extern "C" -int _nc_close_wait(int ncid) -{ - int rc=NSSI_OK; - nssi_request *req=NULL; - request_list_iterator_t iter; - - log_debug(netcdf_debug_level, "enter"); - - iter = file_state_map[ncid]->outstanding_close_requests.begin(); - for (;iter != file_state_map[ncid]->outstanding_close_requests.end();iter++) { - req = *iter; - if (req != NULL) { - nssi_wait(req, &rc); - free(req); - } - } - file_state_map[ncid]->outstanding_close_requests.clear(); - - /* remove data structures associated with ncid */ - if (group_map.find(ncid) != group_map.end()) { - delete group_map[ncid]; - group_map.erase(ncid); - } - - file_state_map[ncid]->mode = NC_CLOSED; - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_close_wait(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_close_wait(ncid); -} - -/** - * Close an open netCDF dataset. - */ -extern "C" -int _nc_close(int ncid) -{ - int rc=NSSI_OK; - nssi_request *req=NULL; - - log_debug(netcdf_debug_level, "enter"); - - if (collective_op_rank == 0) { - req=(nssi_request *)calloc(1, sizeof(nssi_request)); - /* call the remote method */ - rc = nssi_call_rpc(&svcs[default_svc], - NETCDF_CLOSE_OP, - &ncid, - NULL, - 0, - NULL, - req); - - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } else { - MPI_Bcast(&rc, 1, MPI_INT, 0, ncmpi_collective_op_comm); - } - if (rc != NSSI_OK) { - log_error(netcdf_debug_level, "unable to call remote nc_close: %s", - nssi_err_str(rc)); - goto cleanup; - } - - file_state_map[ncid]->mode = NC_ASYNC_CLOSED; - file_state_map[ncid]->outstanding_close_requests.push_back(req); - -#if !defined(USE_ASYNC_CLOSE) - rc=_nc_close_wait(ncid); -#endif - -cleanup: - log_debug(netcdf_debug_level, "exit"); - - return rc; -} -extern "C" -int nc_close(int ncid) -{ - log_debug(netcdf_debug_level, "enter"); - - MANIPULATE_ARGS(ncid); - - return _nc_close(ncid); -} - -extern "C" -int nc_set_file_state(int ncid) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - if (file_state_map[ncid] == NULL) { - file_state_map[ncid]=new_file_state(NC_DEFINE_MODE); - } - MPI_Bcast(&(file_state_map[ncid]->mode), 1, MPI_INT, 0, MPI_COMM_WORLD); - log_debug(netcdf_debug_level, "mode is %d", file_state_map[ncid]->mode); - - if (file_state_map[ncid]->mode == NC_CLOSED) { - log_debug(netcdf_debug_level, "ncid(%d) state is closed. removing file state.", ncid); - if (file_state_map.find(ncid) != file_state_map.end()) { - delete file_state_map[ncid]; - file_state_map.erase(ncid); - } - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -#if DEBUG -int -main (int argc, char *argv[]) -{ - int rc; - int i,j; - struct gengetopt_args_info args_info; - double time; - nssi_remote_pid server_id; - nssi_service xfer_svc; - data_array_t array; - double start_time; - FILE *result_fp = stdout; - log_level debug_level; - nssi_request *reqs; /* the array of requests (for async experiments) */ - data_t *results; /* the array of results (for async experiments) */ - - /* unique to each process */ - int num_reqs; - int myrank=0, np; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &np); - MPI_Comm_rank(MPI_COMM_WORLD, &myrank); - - if (myrank == 0) { - /* Parse command line options to override defaults */ - if (cmdline_parser(argc, argv, &args_info) != 0) { - exit(1); - } - } - - /* send the args to everyone else */ - MPI_Bcast(&args_info, sizeof(struct gengetopt_args_info), MPI_BYTE, - 0, MPI_COMM_WORLD); - - /* initialize logger */ - logger_init(args_info.verbose_arg, NULL); - debug_level = args_info.verbose_arg; - - /* open the result file */ - if (myrank == 0) { - if (args_info.result_file_arg != NULL) { - result_fp = fopen(args_info.result_file_arg, args_info.result_file_mode_arg); - if (result_fp == NULL) { - log_warn(xfer_debug_level, - "invalid result file:" - "defaults to stdout"); - result_fp = stdout; - } - } - } - - /* initialize NSSI RPC */ - nssi_ptl_init(PTL_IFACE_CLIENT, myrank+args_info.server_pid_arg + 1); - nssi_rpc_init(NSSI_RPC_PTL, NSSI_RPC_XDR); - - server_id.nid = args_info.server_nid_arg; - server_id.pid = args_info.server_pid_arg; - - /* If the user did not specify a nid, assume one */ - if (args_info.server_nid_arg == 0) { - nssi_remote_pid myid; - - /* get the id of this client process */ - nssi_get_id(&myid); - - /* assume server is on this host */ - server_id.nid = myid.nid; - } - - print_args(result_fp, &args_info, "%"); - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(1, xfer_1_args, void, void); - NSSI_REGISTER_CLIENT_STUB(2, xfer_2_args, void, void); - - - log_info(debug_level, "\nTrying to get service at (%u,%u)\n", server_id.nid, server_id.pid); - - /* get the service description from the server */ - if (myrank == 0) { - rc = nssi_get_service(server_id, -1, &xfer_svc); - if (rc != NSSI_OK) { - log_error(xfer_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - return rc; - } - } - /* send the args to everyone else */ - MPI_Bcast(&xfer_svc, sizeof(nssi_service), MPI_BYTE, - 0, MPI_COMM_WORLD); - - log_debug(debug_level, "\nConnected to service\n"); - - /* divide the requests among the processors */ - num_reqs = args_info.num_reqs_arg; - - MPI_Barrier(MPI_COMM_WORLD); - - /* allocate requests and results for async reqs */ - reqs = (nssi_request *)calloc(num_reqs, sizeof(nssi_request)); - results = (data_t *)calloc(num_reqs, sizeof(data_t)); - - /* initialize the data array */ - array.data_array_t_len = args_info.len_arg; - array.data_array_t_val = (data_t *)malloc(args_info.len_arg*sizeof(data_t)); - if (array.data_array_t_val == NULL) { - log_error(xfer_debug_level, "out of space"); - return -1; - } - - /* initialize the data */ - for (i=0; i -#include -#include -#include -#include - -#include "ezxml.h" -#include "Trios_logger.h" -#include "Trios_nssi_types.h" -#include "Trios_nssi_fprint_types.h" - -#include "netcdf_config_parser.h" -#include "netcdf_debug.h" - - -/* debug level for the configuration parser */ -log_level netcdf_config_debug_level = LOG_UNDEFINED; - - -/* The config parser parses an NSSI config file (xml) - * and returns the nssi_config data structure. - */ -static int -parse_service(ezxml_t node, char *url) -{ - int rc = NSSI_OK; - const char *attr; - - url[0]='\0'; - /* get the URL attribute */ - attr = ezxml_attr(node, "url"); - if (attr != NULL) { - strncpy(url, attr, NNTI_URL_LEN); - } - - log_debug(netcdf_config_debug_level, "url=%s", url); - - return rc; -} - - -static int -parse_write_type(ezxml_t node, write_type *write_type) -{ - int rc = NSSI_OK; - const char *attr; - - attr = ezxml_attr(node, "default"); - if (!strcmp(attr, "WRITE_DIRECT")) { - *write_type = WRITE_DIRECT; - log_debug(netcdf_config_debug_level, "using %s", attr); - } else if (!strcmp(attr, "WRITE_AGGREGATE_INDEPENDENT")) { - *write_type = WRITE_AGGREGATE_INDEPENDENT; - log_debug(netcdf_config_debug_level, "using %s", attr); - } else if (!strcmp(attr, "WRITE_AGGREGATE_COLLECTIVE")) { - *write_type = WRITE_AGGREGATE_COLLECTIVE; - log_debug(netcdf_config_debug_level, "using %s", attr); - } else if (!strcmp(attr, "WRITE_CACHING_INDEPENDENT")) { - *write_type = WRITE_CACHING_INDEPENDENT; - log_debug(netcdf_config_debug_level, "using %s", attr); - } else if (!strcmp(attr, "WRITE_CACHING_COLLECTIVE")) { - *write_type = WRITE_CACHING_COLLECTIVE; - log_debug(netcdf_config_debug_level, "using %s", attr); - } - - return rc; -} - -static int -parse_num_participants(ezxml_t node, unsigned int *num_participants) -{ - int rc = NSSI_OK; - const char *attr; - - attr = ezxml_attr(node, "default"); - sscanf(attr, "%u", num_participants); - - return rc; -} - -static int -parse_bytes_per_server(ezxml_t node, size_t *bytes_per_server) -{ - int rc = NSSI_OK; - const char *attr; - - attr = ezxml_attr(node, "default"); - sscanf(attr, "%ld", bytes_per_server); - log_debug(netcdf_debug_level, "bytes_per_server %ld", *bytes_per_server); - - return rc; -} - -static int -parse_use_subchunking(ezxml_t node, size_t *use_subchunking) -{ - int rc = NSSI_OK; - const char *attr; - - attr = ezxml_attr(node, "default"); - sscanf(attr, "%ld", use_subchunking); - log_debug(netcdf_debug_level, "use_subchunking %ld", *use_subchunking); - - return rc; -} - -static int -parse_netcdf( - ezxml_t node, - const int index, - struct netcdf_config *config) -{ - int rc = NSSI_OK; - ezxml_t server_id, write_type, num_participants, bytes_per_server, use_subchunking; - - /* server list */ - server_id = ezxml_child(node, "server-id"); - if (server_id) { - rc = parse_service(server_id, (config->netcdf_server_urls[index])); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, - "error parsing serverlist: %s", - nssi_err_str(rc)); - return rc; - } - } - - write_type = ezxml_child(node, "write-type"); - if (write_type) { - rc = parse_write_type(write_type, &config->write_type); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, - "error parsing write_type: %s", - nssi_err_str(rc)); - return rc; - } - } - - num_participants = ezxml_child(node, "num-participants"); - if (num_participants) { - rc = parse_num_participants(num_participants, &config->num_participants); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, - "error parsing num_participants: %s", - nssi_err_str(rc)); - return rc; - } - } - - bytes_per_server = ezxml_child(node, "bytes-per-server"); - if (bytes_per_server) { - rc = parse_bytes_per_server(bytes_per_server, &config->bytes_per_server); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, - "error parsing bytes_per_server: %s", - nssi_err_str(rc)); - return rc; - } - } - - use_subchunking = ezxml_child(node, "use-subchunking"); - if (use_subchunking) { - rc = parse_use_subchunking(use_subchunking, &config->use_subchunking); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, - "error parsing use_subchunking: %s", - nssi_err_str(rc)); - return rc; - } - } - - return rc; -} - - - -static int -parse_config( - ezxml_t node, - struct netcdf_config *config) -{ - int rc; - int i; - int count=0; - ezxml_t service; - - /* process all the storage servers */ - - /* first count the children */ - for (service = ezxml_child(node, "netcdf"); service; service = service->next) - { - count++; - } - - config->num_servers = count; - - config->netcdf_server_urls = (char **)calloc(count, sizeof(char *)); - if (!config->netcdf_server_urls) { - log_error(netcdf_config_debug_level, "could not allocate netcdf services"); - return NSSI_ENOMEM; - } - - for (i=0;inetcdf_server_urls[i] = (char *)calloc(NNTI_URL_LEN, sizeof(char)); - if (!config->netcdf_server_urls[i]) { - log_error(netcdf_config_debug_level, "could not allocate netcdf services"); - return NSSI_ENOMEM; - } - } - - - /* initialize count again */ - count = 0; - - /* parse the service list */ - for (service = ezxml_child(node, "netcdf"); service; service = service->next) - { - if (service) { - rc = parse_netcdf(service, count++, config); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, "error parsing netcdf service: %s", - nssi_err_str(rc)); - return rc; - } - } - } - - return rc; -} - - -/*------------------ EXTERNAL APIs ------------------ */ - -int -parse_netcdf_config_file( - const char *docname, - struct netcdf_config *netcdf_cfg) -{ - int rc = NSSI_OK; - int fd = 0; - ezxml_t doc, config; - - log_debug(netcdf_config_debug_level, "entered parse_config_file"); - - if (!docname) { - rc = NSSI_EINVAL; - return rc; - } - - memset(netcdf_cfg, 0, sizeof(struct netcdf_config)); - - fd = open(docname, O_RDONLY, 0); - if (fd < 0) { - log_error(netcdf_config_debug_level, "failed to open config file (%s): %s", docname, strerror(errno)); - return NSSI_EIO; - } - doc = ezxml_parse_fd(fd); - close(fd); - if (doc == NULL) { - log_error(netcdf_config_debug_level, "failed to parse config file (%s)", docname); - return NSSI_EINVAL; - } - - config = ezxml_child(doc, "config"); - if (config) { - rc = parse_config(config, netcdf_cfg); - if (rc != NSSI_OK) { - log_error(netcdf_config_debug_level, - "could not parse config file: %s", - nssi_err_str(rc)); - return rc; - } - } - - ezxml_free(doc); - - log_debug(netcdf_config_debug_level, "finished parse_config_file"); - - return rc; -} - - -void netcdf_config_free(struct netcdf_config *netcdf_cfg) -{ - int i; - - /* release the space allocated for the ss_server_ids */ - for (i=0;inum_servers;i++) { - free(netcdf_cfg->netcdf_server_urls[i]); - } - free(netcdf_cfg->netcdf_server_urls); -} - -#endif // HAVE_TRIOS_PNETCDF diff --git a/packages/trios/services/netcdf/src/netcdf_config_parser.h b/packages/trios/services/netcdf/src/netcdf_config_parser.h deleted file mode 100644 index 154a5294b8b9..000000000000 --- a/packages/trios/services/netcdf/src/netcdf_config_parser.h +++ /dev/null @@ -1,108 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file config_parser.h - * - * @brief Parse the nssi configuration XML file. - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1.23 $ - * $Date: 2005/11/09 20:15:51 $ - * - */ - -#ifndef _NETCDF_CONFIG_PARSER_H_ -#define _NETCDF_CONFIG_PARSER_H_ - -#include "Trios_nssi_types.h" -#include "netcdf_args.h" - -#ifdef __cplusplus -extern "C" { -#endif - - extern log_level config_debug_level; - - /** - * @brief A structure to represent the configuration of - * NSSI core services. - */ - struct netcdf_config { - - /** @brief Number of available storage servers */ - int num_servers; - - /** @brief storage service IDs */ - char **netcdf_server_urls; - - /** @brief number of clients the server can expect */ - unsigned int num_participants; - - /** @brief The type of write operation the client wishes the server to perform */ - enum write_type write_type; - - /** @brief The number of bytes the server can cache for aggregation */ - size_t bytes_per_server; - - /** @brief The number of bytes the server can cache for aggregation */ - size_t use_subchunking; - - }; - - - -#if defined(__STDC__) || defined(__cplusplus) - - extern int parse_netcdf_config_file(const char *fname, - struct netcdf_config *config); - - extern void netcdf_config_free( - struct netcdf_config *config); - -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/services/netcdf/src/netcdf_debug.c b/packages/trios/services/netcdf/src/netcdf_debug.c deleted file mode 100644 index fa59fafd5f18..000000000000 --- a/packages/trios/services/netcdf/src/netcdf_debug.c +++ /dev/null @@ -1,46 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include -#include "netcdf_debug.h" - -log_level netcdf_debug_level = LOG_UNDEFINED; diff --git a/packages/trios/services/netcdf/src/netcdf_debug.h b/packages/trios/services/netcdf/src/netcdf_debug.h deleted file mode 100644 index 35a67284d881..000000000000 --- a/packages/trios/services/netcdf/src/netcdf_debug.h +++ /dev/null @@ -1,78 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/*-------------------------------------------------------------------------*/ -/** - * @file netcdf_debug.h - * - * @author Ron Oldfield (raoldfi\@sandia.gov) - * $Revision: 1640 $ - * $Date: 2007-11-28 11:59:53 -0700 (Wed, 28 Nov 2007) $ - * - */ - -#ifndef _NETCDF_DEBUG_H_ -#define _NETCDF_DEBUG_H_ - -#include "Trios_logger.h" - -#ifdef __cplusplus -extern "C" { -#endif - - extern log_level netcdf_debug_level; - -#define NC_ENOTNC4 (-111) -#define NC_ENOGRP (-125) - - -#if defined(__STDC__) || defined(__cplusplus) - -#else /* K&R C */ - -#endif /* K&R C */ - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/packages/trios/services/netcdf/src/netcdf_server.cpp b/packages/trios/services/netcdf/src/netcdf_server.cpp deleted file mode 100644 index a1c71db71e02..000000000000 --- a/packages/trios/services/netcdf/src/netcdf_server.cpp +++ /dev/null @@ -1,2398 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file main.c - * - * @brief Driver for the NSSI name server. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - * $Revision: 1264 $. - * $Date: 2007-02-27 15:30:26 -0700 (Tue, 27 Feb 2007) $. - */ - - -#include "Trios_config.h" - -#ifdef HAVE_TRIOS_PNETCDF - -#ifndef __STDC_CONSTANT_MACROS -#define __STDC_CONSTANT_MACROS -#endif - -#include -#include -#include -#include - -#include /* for nssi service functions */ -#include -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include "pnetcdf.h" -#include -#include - -using namespace std; - -#include "netcdf_args.h" -#include "netcdf_config.h" -#include "netcdf_debug.h" /* netcdf_debug_level */ - -#include "aggregation.h" - -#include "Trios_timer.h" - - -typedef char NNTI_url[NNTI_URL_LEN]; - - -#ifdef __LIBCATAMOUNT__ -#define ntohs(value) 0 -#endif - - -static double create_time=0.0; -static double open_time =0.0; -static double enddef_time=0.0; -static double close1_time=0.0; -static double close2_time=0.0; - - -/* -------------------- PRIVATE FUNCTIONS ---------- */ - - -static int do_put_vars(int ncid, int varid, - const MPI_Offset start[], - const MPI_Offset count[], - const MPI_Offset stride[], - const void *buf, MPI_Offset bufcount, - MPI_Datatype datatype) -{ - int rc=NC_NOERR; - - log_debug(netcdf_debug_level, "calling ncmpi_put_vars with direct data"); - rc = ncmpi_put_vars(ncid, varid, start, count, stride, - buf, bufcount, datatype); - - return rc; -} - -static int do_put_vars(aggregation_chunk_details_t **chunks, const int chunk_count) -{ - int rc=NC_NOERR; - trios_declare_timer(call_time); - - for (int i=0;incid, - chunks[i]->varid, - chunks[i]->start, - chunks[i]->count, - chunks[i]->stride, - chunks[i]->buf, - chunks[i]->num_elements, - chunks[i]->datatype); - trios_stop_timer("ncmpi_put_vars", call_time); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - goto cleanup; - } - } - -cleanup: - return rc; -} - -static int do_put_vars_all(const int ncid, const int varid, aggregation_chunk_details_t **chunks, int chunk_count) -{ - int rc=NC_NOERR; - int min_chunk_count=0; - int max_chunk_count=0; - - /* - * max_chunk_count is the largest number of chunks held by any server. this will - * determine is anyone has chunks for this varid. - */ - MPI_Allreduce(&chunk_count, &max_chunk_count, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); - - if (max_chunk_count == 0) { - // no one has any chunks for this varid. bail out. - return NC_NOERR; - } - - /* - * min_chunk_count is the smallest number of chunks held by any server. this is - * the maximum number of collective calls allowed. remaining chunks will be put - * in independent data mode. - */ - MPI_Allreduce(&chunk_count, &min_chunk_count, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - - for (int i=0;incid, chunks[i]->varid, chunks[i]->ndims, - chunks[i]->start[0], chunks[i]->start[1], chunks[i]->start[2], chunks[i]->start[3], - chunks[i]->count[0], chunks[i]->count[1], chunks[i]->count[2], chunks[i]->count[3], - chunks[i]->num_elements); - rc = ncmpi_put_vars_all(chunks[i]->ncid, - chunks[i]->varid, - chunks[i]->start, - chunks[i]->count, - chunks[i]->stride, - chunks[i]->buf, - chunks[i]->num_elements, - chunks[i]->datatype); - - log_debug(netcdf_debug_level, "ncmpi_put_vars_all: rc==%d", rc); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - goto cleanup; - } - } - - if (min_chunk_count < chunk_count) { - ncmpi_begin_indep_data(ncid); - for (int i=min_chunk_count;incid, - chunks[i]->varid, - chunks[i]->start, - chunks[i]->count, - chunks[i]->stride, - chunks[i]->buf, - chunks[i]->num_elements, - chunks[i]->datatype); - - log_debug(netcdf_debug_level, "ncmpi_put_vars: rc==%d", rc); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - goto cleanup; - } - } - ncmpi_end_indep_data(ncid); - } - -cleanup: - return rc; -} - -static int flush_aggregated_chunks( - const int ncid, - const int varid) -{ - int rc=NC_NOERR; - int chunk_count=0; - trios_declare_timer(callTime); - - trios_start_timer(callTime); - try_aggregation(ncid, varid); - trios_stop_timer("agg", callTime); - - aggregation_chunk_details_t **chunks = get_chunks(ncid, varid, &chunk_count); - if (chunk_count == 0) { - log_debug(netcdf_debug_level, "chunk_count is 0, but I must participate in the collective."); - } else { - log_debug(netcdf_debug_level, "chunk_count is %d.", chunk_count); - } - - if (use_collective(ncid)) { - trios_start_timer(callTime); - rc = do_put_vars_all(ncid, varid, chunks, chunk_count); - if (rc == NC_EINDEP) { - rc = do_put_vars(chunks, chunk_count); - } - trios_stop_timer("put_vars_all", callTime); - - free(chunks); - cleanup_aggregation_chunks(ncid, varid); - - log_debug(netcdf_debug_level, "do_put_vars_all: rc==%d", rc); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - } - } else { - trios_start_timer(callTime); - rc = do_put_vars(chunks, chunk_count); - if (rc == NC_ENOTINDEP) { - rc = do_put_vars_all(ncid, varid, chunks, chunk_count); - } - trios_stop_timer("put_vars", callTime); - - free(chunks); - cleanup_aggregation_chunks(ncid, varid); - - log_debug(netcdf_debug_level, "do_put_vars: rc==%d", rc); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - } - } - - return rc; -} -static int flush_aggregated_chunks(const int ncid) -{ - int rc=NC_NOERR; - int chunk_count=0; - trios_declare_timer(callTime); - - if (use_aggregation(ncid)) { - log_debug(netcdf_debug_level, "flushing aggregation chunks"); - - if (use_collective(ncid)) { - log_debug(netcdf_debug_level, "using collective"); - // get num vars - // iter thru vars - // - try_agg on varid - // - get_chunks - // - do_put_vars_all - - int var_count=0; - int varid=0; - - /* get varids for this group */ - if ((rc = ncmpi_inq_nvars(ncid, &var_count)) != NC_NOERR) { - log_error(netcdf_debug_level, "Could not get varids: %s", ncmpi_strerror(rc)); - goto cleanup; - } - for (int i=0;i=', is the full name of the hint - * to be set, and is the hint value. - * E.g., 'setenv MPIIO_HINT__panfs_concurrent_write 1' - * or 'MPIIO_HINT__panfs_concurrent_write=1' in the hints file. - */ - MPI_Info_create(mpiHints); - - /* get hints from environment */ - for (i = 0; environ[i] != NULL; i++) { - /* if this is an MPIIO_HINT, pass the hint to the info object */ - if (strncmp(environ[i], "MPIIO_HINT", strlen("MPIIO_HINT")) == 0) { - strcpy(hintString, environ[i]); - ExtractHint(settingVal, valueVal, hintString); - MPI_Info_set(*mpiHints, settingVal, valueVal); - } - } - - /* get hints from hints file */ - if (strcmp(hintsFileName, "") != 0) { - - /* open the hint file */ - fd = fopen(hintsFileName, "r"); - if (fd == NULL) { - log_error(netcdf_debug_level, "cannot open hints file"); - } else { - /* iterate over hints file */ - while(fgets(hintString, MAX_HINT_STR, fd) != NULL) { - if (strncmp(hintString, "MPIIO_HINT", strlen("MPIIO_HINT")) == 0) { - ExtractHint(settingVal, valueVal, hintString); - MPI_Info_set(*mpiHints, settingVal, valueVal); - } - } - /* close the hints files */ - if (fclose(fd) != 0) log_error(netcdf_debug_level, "cannot close hints file"); - } - } -} /* SetHints() */ - - -/******************************************************************************/ -/* - * Show all hints (key/value pairs) in an MPI_Info object. - */ - -void ShowHints(MPI_Info * mpiHints) -{ - char key[MPI_MAX_INFO_VAL], - value[MPI_MAX_INFO_VAL]; - int flag, - i, - nkeys; - - MPI_Info_get_nkeys(*mpiHints, &nkeys); - - for (i = 0; i < nkeys; i++) { - MPI_Info_get_nthkey(*mpiHints, i, key); - MPI_Info_get(*mpiHints, key, MPI_MAX_INFO_VAL-1, value, &flag); - log_debug(netcdf_debug_level, "mpiHint[%d]: %s = %s", i, key, value); - } -} /* ShowHints() */ - -/* -------------------- SERVER-SIDE STUBS ---------- */ - -/** - * @brief Create a netcdf dataset. - * - * This is the server-side stub that uses nc_create to create - * a netcdf dataset. It returns the - */ -int nc_create_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_create_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - int rank, np; - const char *path = args->path; - const int cmode = args->cmode; - size_t chunksizehint = args->chunksizehint; - int ncid=-1; - nc_create_res res; /* this is what we send back to the client */ - MPI_Info mpiHints = MPI_INFO_NULL; - - log_level debug_level = netcdf_debug_level; - - log_debug(debug_level, "calling nc_create(%s, %d, %d)", path, cmode, ncid); - - create_time=MPI_Wtime(); - - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - SetHints(&mpiHints, (char*)""); - ShowHints(&mpiHints); - - trios_declare_timer(callTime); - trios_start_timer(callTime); - rc = ncmpi_create(MPI_COMM_WORLD, path, cmode, mpiHints, &ncid); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not create file(%s): %s", path, ncmpi_strerror(rc)); - goto cleanup; - } - trios_stop_timer("create", callTime); - - /* set the result structure */ - memset(&res, 0, sizeof(res)); - res.chunksizehint = chunksizehint; - res.ncid = ncid; - - if (args->num_participants > 0) { - set_participant_count(ncid, caller, args->write_type, args->num_participants); - } else { - add_participant_for_file(ncid, caller, args->write_type); - } - -cleanup: - log_debug(netcdf_debug_level, "exit (ncid=%d)", ncid); - - /* send the ncipd and return code back to client */ - rc = nssi_send_result(caller, request_id, rc, &res, res_addr); - - return rc; -} - -/** - * Get the metadata associated a complete group hierarchy. - * - * A group consists of an id, name, dimensions (for all group vars), - * and attributes. This function gathers each of these attributes - * through the nc_inq_* commands, then recursively gets the attributes - * for each of the subgroups. - */ -int get_group_metadata( - const int ncid, - struct nc_group *group) -{ - int rc = NC_NOERR, natts; - log_level debug_level = netcdf_debug_level; - - int *dimids=NULL; - int *varids=NULL; - int *groupids = NULL; - - /* set the ncid */ - group->ncid = ncid; - - -#ifdef USING_NETCDF4 - - /* get the namelen for the name of this group */ - if ((rc = nc_inq_grpname_len(ncid, &namelen)) != NC_NOERR) { - log_error(debug_level, "Could not get namelen: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* get the name */ - group->name = (char *)calloc(namelen+1, sizeof(char)); - if ((rc = nc_inq_grpname_full(ncid, &namelen, group->name)) != NC_NOERR) { - log_error(debug_level, "Could not get groupname: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* get parent ncid */ - rc = nc_inq_grp_parent(ncid, &group->parent_ncid); - if (rc == NC_ENOTNC4) { - group->parent_ncid = -1; - rc = NC_NOERR; - } - else if (rc != NC_NOERR) { - log_error(debug_level, "Could not get parent ncid: %s", ncmpi_strerror(rc)); - goto cleanup; - } - -#else - - /* Without group support (in NC4), there is only one group */ - group->name = strdup("/"); - group->parent_ncid = -1; - - - /* get the unlimdimid */ - if ((rc = ncmpi_inq_unlimdim(ncid, &group->unlimdimid)) != NC_NOERR) { - log_error(debug_level, "Error getting unlimdimid: %s", ncmpi_strerror(rc)); - goto cleanup; - } - -#endif - - /* ----- Get Dimension Metadata for the group ----- */ - dimids = (int *)calloc(NC_MAX_DIMS, sizeof(int)); - -#ifdef USING_NETCDF4 - /* get the dimids for this group */ - if ((rc = nc_inq_dimids(ncid, (int *)&group->dims.dims_len, dimids, 0)) != NC_NOERR) { - log_error(debug_level, "Error getting dimids: %s", ncmpi_strerror(rc)); - goto cleanup; - } -#else - /* get the number of dimids for this dataset */ - if ((rc = ncmpi_inq_ndims(ncid, (int *)&group->dims.dims_len)) != NC_NOERR) { - log_error(debug_level, "Error getting dimids: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* in netcdf3, dimids range from 0 -- (ndims-1) */ - for (uint32_t i=0; idims.dims_len; i++) { - dimids[i] = i; - } -#endif - - log_debug(debug_level, "got ndims=%d (ncid=%d)", group->dims.dims_len, ncid); - - /* get metadata for each dimension */ - group->dims.dims_val = (nc_dim *)calloc(group->dims.dims_len, sizeof(nc_dim)); - for (uint32_t i=0; idims.dims_len; i++) { - - /* set current dimension */ - struct nc_dim *dim = &group->dims.dims_val[i]; - dim->dimid = dimids[i]; - - /* allocate space for name (assume largest) */ - dim->name = (char *)calloc(NC_MAX_NAME, sizeof(char)); - - /* get dimension metadata */ - rc = ncmpi_inq_dim(ncid, dim->dimid, dim->name, (MPI_Offset*)&dim->len); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not get dimid=%d metadata: %s", dimids[i], ncmpi_strerror(rc)); - goto cleanup; - } - } - - /* free the dimids, we will use them again for variables */ - free(dimids); - dimids = NULL; - - /* ------- Global attribute metadata -------- */ - ncmpi_inq_natts(ncid, &natts); - group->atts.atts_len = natts; - group->atts.atts_val = (nc_att *)calloc(natts, sizeof(nc_att)); - - for (int32_t i=0; i< natts; i++) { - nc_att *att = &group->atts.atts_val[i]; - att->name = (char *)calloc(NC_MAX_NAME, sizeof(char)); - rc = ncmpi_inq_attname(ncid, NC_GLOBAL, i, att->name); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not get global attributes for ncid=%d", ncid); - goto cleanup; - } - rc = ncmpi_inq_att(ncid, NC_GLOBAL, att->name, (nc_type *)&att->xtype, (MPI_Offset*)&att->len); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not get global attributes for ncid=%d", ncid); - goto cleanup; - } - log_debug(netcdf_debug_level, "attr: name(%s) type(%d) len(%d)", att->name, att->xtype, att->len); - } - - log_debug(debug_level, "Got attribute metadata"); - - /* ----- Get Variable Metadata ----- */ - varids = (int *)calloc(NC_MAX_VARS, sizeof(int)); - -#ifdef USING_NETCDF4 - /* get varids for this group */ - if ((rc = nc_inq_varids(ncid, (int *)&group->vars.vars_len, varids)) != NC_NOERR) { - log_error(debug_level, "Could not get varids: %s", ncmpi_strerror(rc)); - goto cleanup; - } -#else - /* get varids for this group */ - if ((rc = ncmpi_inq_nvars(ncid, (int *)&group->vars.vars_len)) != NC_NOERR) { - log_error(debug_level, "Could not get varids: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* in netcdf3, varids ranged from 0 to (nids-1) */ - for (uint32_t i=0; ivars.vars_len; i++) { - varids[i] = i; - } -#endif - - /* allocate space for variables */ - group->vars.vars_val = (nc_var *)calloc(group->vars.vars_len, sizeof(nc_var)); - - log_debug(debug_level, "Got varids (%d vars)", (int)group->vars.vars_len); - - /* get metadata about each variable */ - for (uint32_t i=0; ivars.vars_len; i++) { - - /* Variable structure for this ID */ - struct nc_var *var = &group->vars.vars_val[i]; - var->varid = varids[i]; - - /* First we need to allocate space for name and the dimids */ - var->name = (char *)calloc(NC_MAX_NAME, sizeof(char)); - var->dimids.dimids_val = (int *)calloc(NC_MAX_VAR_DIMS, sizeof(int)); - - /* Get metadata for the variable */ - rc = ncmpi_inq_var(ncid, var->varid, var->name, (nc_type *)&var->xtype, - (int *)&var->dimids.dimids_len, var->dimids.dimids_val, - (int *)&var->atts.atts_len); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not call nc_inq_var(varid=%d): %s", - var->varid, ncmpi_strerror(rc)); - goto cleanup; - } - - log_debug(debug_level, "Got info for var[%d], var->varid=%d, var->name=%s", i, var->varid, var->name); - - /* Get the attributes */ - var->atts.atts_val = (nc_att *)calloc(var->atts.atts_len, sizeof(nc_att)); - for (uint32_t j = 0; jatts.atts_len; j++) { - - struct nc_att *att = &var->atts.atts_val[j]; - - /* get attribute name */ - att->name = (char *)calloc(NC_MAX_NAME, sizeof(char)); - if ((rc = ncmpi_inq_attname(ncid, var->varid, j, att->name)) != NC_NOERR) { - log_error(debug_level, "Could not get attribute name: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* get attribute info */ - if ((rc = ncmpi_inq_att(ncid, var->varid, att->name, (nc_type *)&att->xtype, (MPI_Offset*)&att->len)) != NC_NOERR) { - log_error(debug_level, "Could not get attribute metadata: %s", ncmpi_strerror(rc)); - goto cleanup; - } - } - } - - log_debug(debug_level, "Got variable metadata"); - -#ifdef USING_NETCDF4 - - /* Get number of subgroups -- WHY NOT HAVE NC_INQ_NGRPS? */ - rc = nc_inq_grps(ncid, (int *)&group->groups.groups_len, NULL); - if (rc == NC_ENOTNC4) { - group->groups.groups_len = 0; - rc = NC_NOERR; - } - else if (rc != NC_NOERR) { - log_error(debug_level, "Unable to get grouplen: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - log_debug(debug_level, "Got number of subgroups (%d)", (int)group->groups.groups_len); - - if (group->groups.groups_len > 0) { - /* Get group IDs */ - groupids = (int *)calloc(group->groups.groups_len, sizeof(int)); - if ((rc = nc_inq_grps(ncid, NULL, groupids)) != NC_NOERR) { - log_error(debug_level, "Unable to get grouplen: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* Recursively get subgroup metadata */ - group->groups.groups_val = (nc_group *)calloc(group->groups.groups_len, sizeof(nc_group)); - for (int32_t i=0; igroups.groups_len; i++) { - if ((rc = get_group_metadata(groupids[i], &group->groups.groups_val[i])) != NC_NOERR) { - log_error(debug_level, "Unable to get subgroup metadata: %s", ncmpi_strerror(rc)); - goto cleanup; - } - } - - log_debug(debug_level, "Got all subgroup metadata"); - } -#endif - - -cleanup: - if (dimids != NULL) free(dimids); - if (varids != NULL) free(varids); - if (groupids != NULL) free(groupids); - - return rc; -} - - -/** - * @brief Open a netcdf dataset. - * - * Open a netcdf dataset and return all associated metadata. - */ -int nc_open_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_open_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - const char *path = args->path; - const int omode = args->mode; - size_t chunksizehint = args->chunksizehint; - int ncid=-1; - nc_open_res res; /* this is what we send back to the client */ - MPI_Info mpiHints = MPI_INFO_NULL; - - memset(&res, 0, sizeof(res)); - - log_level debug_level = netcdf_debug_level; - - log_debug(debug_level, "calling nc__open(%s, %d)", path, omode); - - open_time=MPI_Wtime(); - - SetHints(&mpiHints, (char*)""); - ShowHints(&mpiHints); - - trios_declare_timer(callTime); - trios_start_timer(callTime); - if ((rc = ncmpi_open(MPI_COMM_WORLD, path, omode, mpiHints, &ncid)) != NC_NOERR) { - log_error(debug_level, "Error opening file \"%s\": %s", path, ncmpi_strerror(rc)); - res.root_group.ncid = -1; - goto cleanup; - } - trios_stop_timer("open", callTime); - - log_debug(debug_level, "dataset open (ncid=%d)", ncid); - - /* set the result structure */ - res.chunksizehint = chunksizehint; - res.root_group.ncid = ncid; - - rc = get_group_metadata(ncid, &res.root_group); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not get root_group metadata"); - goto cleanup; - } - - if (args->num_participants > 0) { - set_participant_count(ncid, caller, args->write_type, args->num_participants); - } else { - add_participant_for_file(ncid, caller, args->write_type); - } - -cleanup: - log_debug(netcdf_debug_level, "exit (ncid=%d)", ncid); - - /* send the ncid and return code back to client */ - rc = nssi_send_result(caller, request_id, rc, &res, res_addr); - - return rc; -} - -/** - * @brief Define a dimension of a dataset. - * - * This is the server-side stub that uses nc_create to create - * a netcdf dataset. It returns the - */ -int nc_def_dim_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_def_dim_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - const int ncid = args->ncid; - const char *name = args->name; - const size_t len = args->len; - int dimid; /* result */ - - log_debug(netcdf_debug_level, "enter"); - - trios_declare_timer(callTime); - trios_start_timer(callTime); - /* call actual netcdf function */ - rc = ncmpi_def_dim(ncid, name, len, &dimid); - trios_stop_timer("def dim", callTime); - - log_debug(netcdf_debug_level, "nc_def_dim: rc=%s", ncmpi_strerror(rc)); - - /* send the ncipd and return code back to client */ - rc = nssi_send_result(caller, request_id, rc, &dimid, res_addr); - - log_debug(netcdf_debug_level, "finished nc_def_dim(ncid=%d, name=%s, len=%d, dimid=%d)", - ncid, name, len, dimid); - - log_debug(netcdf_debug_level, "exit (dimid=%d)", dimid); - - return rc; -} - -/** - * @brief Define a variable of a dataset. - * - * This is the server-side stub that uses nc_create to create - * a netcdf dataset. It returns the - */ -int nc_def_var_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_def_var_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - log_debug(netcdf_debug_level, "enter"); - - log_debug(netcdf_debug_level, "calling nc_def_var"); - - int rc = NC_NOERR; - const int ncid = args->ncid; - const char *name = args->name; - const nc_type xtype = (nc_type)args->xtype; - const int ndims = args->dimids.dimids_len; - const int *dimids = args->dimids.dimids_val; - int varid; /* result */ - - log_debug(netcdf_debug_level, "calling nc_def_var(ncid=%d, name=\"%s\", xtype=%d, " - "ndims=%d, dimids[0]=%d)", ncid, name, xtype, ndims, dimids[0]); - - - trios_declare_timer(callTime); - trios_start_timer(callTime); - /* call real netcdf function */ - rc = ncmpi_def_var(ncid, name, xtype, ndims, dimids, &varid); - if (rc != NC_NOERR) { - log_error(netcdf_debug_level, "%s", ncmpi_strerror(rc)); - } - trios_stop_timer("def var", callTime); - - /* send the ncipd and return code back to client */ - rc = nssi_send_result(caller, request_id, rc, &varid, res_addr); - - log_debug(netcdf_debug_level, "exit (varid=%d)", varid); - - return rc; -} - -int ncmpi_inq_type( - const int ncid, - const nc_type xtype, - char *name, - MPI_Offset *sizep) -{ - int rc = NC_NOERR; - - log_debug(netcdf_debug_level, "enter"); - - switch (xtype) { - - case NC_BYTE: - *sizep = 1; - if (name) strcpy(name, "NC_BYTE"); - break; - - case NC_CHAR: - *sizep = sizeof(char); - if (name) strcpy(name, "NC_CHAR"); - break; - - case NC_SHORT: - *sizep = sizeof(short); - if (name) strcpy(name, "NC_SHORT"); - break; - - case NC_INT: - *sizep = sizeof(int); - if (name) strcpy(name, "NC_INT"); - break; - - case NC_FLOAT: - *sizep = sizeof(float); - if (name) strcpy(name, "NC_FLOAT"); - break; - - case NC_DOUBLE: - *sizep = sizeof(double); - if (name) strcpy(name, "NC_DOUBLE"); - break; - - default: - rc = NC_EBADTYPE; - break; - } - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -int nc_get_att_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_get_att_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - log_level debug_level = netcdf_debug_level; - - const int ncid = args->ncid; - const int varid = args->varid; - const char *name = args->name; - - nc_type xtype; - MPI_Offset len=0, nbytes=0, typesize=0; - void *attbuf=NULL; - - log_debug(netcdf_debug_level, "enter"); - - /* get the len and type of the attribute buffer */ - rc = ncmpi_inq_att(ncid, varid, name, &xtype, &len); - if (rc != NC_NOERR) { - log_error(debug_level, "Unable to inq_att: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - /* get the size of the datatype */ - rc = ncmpi_inq_type(ncid, xtype, NULL, &typesize); - if (rc != NC_NOERR) { - log_error(debug_level, "Unable to inq_type: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - nbytes = len * typesize; - attbuf = calloc(nbytes, sizeof(char)); - - switch (xtype) { - - case NC_CHAR: - rc = ncmpi_get_att_text(ncid, varid, name, (char *)attbuf); - break; - - case NC_SHORT: - rc = ncmpi_get_att_short(ncid, varid, name, (short *)attbuf); - break; - - case NC_INT: - rc = ncmpi_get_att_int(ncid, varid, name, (int *)attbuf); - break; - - case NC_FLOAT: - rc = ncmpi_get_att_float(ncid, varid, name, (float *)attbuf); - break; - - case NC_DOUBLE: - rc = ncmpi_get_att_double(ncid, varid, name, (double *)attbuf); - break; - - default: - rc = NC_EIOMISMATCH; - break; - - } - - if (rc != NC_NOERR) { - log_error(debug_level, "%s", ncmpi_strerror(rc)); - goto cleanup; - } - - - /* send the buf to the client */ - rc = nssi_put_data(caller, attbuf, nbytes, data_addr, -1); - if (rc != NSSI_OK) { - log_error(debug_level, "%s", nssi_err_str(rc)); - goto cleanup; - } - -cleanup: - - /* send result back to client */ - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - - if (attbuf) free(attbuf); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -int nc_put_att_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_put_att_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - - const int ncid = args->ncid; - const int varid = args->varid; - const char *name = args->name; - const nc_type xtype = (nc_type)args->xtype; /* one of the 6 netcdf xtypes */ - const arg_type atype = args->atype; - const ssize_t len = args->data.data_len; - const char *data = args->data.data_val; - - log_debug(netcdf_debug_level, "enter"); - - - /* atype (arg type) identifies the function to call */ - switch(atype) { - - case NC_ARG_TEXT: - { - log_debug(netcdf_debug_level, "calling nc_put_att_text(ncid=%d, varid=%d, " - "name=%s, len=%d, data=%s)", ncid, varid, name, len, data); - - rc = ncmpi_put_att_text(ncid, varid, name, - len/sizeof(char), data); - break; - } - - case NC_ARG_UCHAR: - { - rc = ncmpi_put_att_uchar(ncid, varid, name, xtype, - len/sizeof(unsigned char), (const unsigned char *)data); - break; - } - - case NC_ARG_SCHAR: - { - rc = ncmpi_put_att_schar(ncid, varid, name, xtype, - len/sizeof(signed char), (const signed char *)data); - break; - } - - case NC_ARG_SHORT: - { - rc = ncmpi_put_att_short(ncid, varid, name, xtype, - len/sizeof(short), (const short *)data); - break; - } - - case NC_ARG_INT: - { - rc = ncmpi_put_att_int(ncid, varid, name, xtype, - len/sizeof(int), (const int *)data); - break; - } - - case NC_ARG_LONG: - { - rc = ncmpi_put_att_long(ncid, varid, name, xtype, - len/sizeof(long), (const long *)data); - break; - } - - case NC_ARG_FLOAT: - { - rc = ncmpi_put_att_float(ncid, varid, name, xtype, - len/sizeof(float), (const float *)data); - break; - } - - case NC_ARG_DOUBLE: - { - rc = ncmpi_put_att_double(ncid, varid, name, xtype, - len/sizeof(double), (const double *)data); - break; - } - - case NC_ARG_UBYTE: - { -#if HAVE_NC_PUT_ATT_UBYTE - rc = ncmpi_put_att_ubyte(ncid, varid, name, xtype, - len/sizeof(unsigned char), (const unsigned char *)data); -#else - rc = NC_ENOTSUPP; /* error added to handle lack of support */ -#endif - break; - } - - case NC_ARG_USHORT: - { -#if HAVE_NC_PUT_ATT_USHORT - rc = ncmpi_put_att_ushort(ncid, varid, name, xtype, - len/sizeof(unsigned short), (const unsigned short *)data); -#else - rc = NC_ENOTSUPP; -#endif - break; - } - - case NC_ARG_UINT: - { -#if HAVE_NC_PUT_ATT_UINT - rc = ncmpi_put_att_uint(ncid, varid, name, xtype, - len/sizeof(unsigned int), (const unsigned int *)data); -#else - rc = NC_ENOTSUPP; -#endif - break; - } - - case NC_ARG_LONGLONG: - { -#if HAVE_NC_PUT_ATT_LONGLONG - rc = ncmpi_put_att_longlong(ncid, varid, name, xtype, - len/sizeof(long long), (const long long *)data); -#else - rc = NC_ENOTSUPP; -#endif - break; - } - - case NC_ARG_ULONGLONG: - { -#if HAVE_NC_PUT_ATT_ULONGLONG - rc = ncmpi_put_att_ulonglong(ncid, varid, name, xtype, - len/sizeof(unsigned long long), (const unsigned long long *)data); -#else - rc = NC_ENOTSUPP; -#endif - break; - } - - case NC_ARG_VOID: - { - //rc = ncmpi_put_att(ncid, varid, name, xtype, len, data); - //break; - } - - default: - rc = NC_EBADTYPE; - } - - - /* send result back to client */ - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - - -/** - * Write a variable to the underlying netcdf file. - * - * This implementation will pull raw data from the client - * using the data. - * - * TODO: Put data in a cache before calling the actual netcdf function. - * - */ -int nc_put_vars_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_put_vars_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - int i, ndims; - - const int ncid = args->ncid; - const int varid = args->varid; - const int atype = args->atype; - const nc_type buftype = (nc_type)args->buftype; - const size_t count = args->element_count; - const size_t len = args->len; - const nc_size_t *startp = args->start.start_val; - const nc_size_t *countp = args->count.count_val; - const nc_size_t *stridep = args->stride.stride_val; - MPI_Offset *count_copy = NULL; - MPI_Offset *start_copy = NULL; - MPI_Offset *stride_copy = NULL; - int *dimids = NULL; - - log_level debug_level = netcdf_debug_level; - - MPI_Datatype datatype; - - size_t ccount = 1; - - void *buf=NULL; - - - log_debug(netcdf_debug_level, "enter"); - - log_debug(debug_level, "calling nc_put_vars_stub(ncid=%d, varid=%d, atype=%d, len=%d)", ncid, varid, atype, len); - - trios_declare_timer(callTime); - - trios_start_timer(callTime); - buf = malloc(len); - trios_stop_timer("buf malloc", callTime); - - trios_start_timer(callTime); - - rc = ncmpi_inq_varndims(ncid, varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not get ndims: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (ndims) { - /* Treat each arg type differently */ - switch (buftype) { - case NC_BYTE: - datatype = MPI_BYTE; - log_debug(debug_level, "using MPI_BYTE (count=%d ; len=%d)", count, len); - break; - case NC_CHAR: - datatype = MPI_CHAR; - log_debug(debug_level, "using MPI_CHAR (count=%d ; len=%d)", count, len); - break; - - case NC_SHORT: - datatype = MPI_SHORT; - log_debug(debug_level, "using MPI_SHORT (count=%d ; len=%d)", count, len); - break; - - case NC_INT: - datatype = MPI_INT; - log_debug(debug_level, "using MPI_INT (count=%d ; len=%d)", count, len); - break; - - case NC_FLOAT: - datatype = MPI_FLOAT; - log_debug(debug_level, "using MPI_FLOAT (count=%d ; len=%d)", count, len); - break; - - case NC_DOUBLE: - datatype = MPI_DOUBLE; - log_debug(debug_level, "using MPI_DOUBLE (count=%d ; len=%d)", count, len); - break; - default: - log_error(debug_level, "Operation=%d not supported", buftype); - rc = NC_ENOTSUPP; - goto cleanup; - } - start_copy = (MPI_Offset *)calloc(ndims, sizeof(MPI_Offset)); - count_copy = (MPI_Offset *)calloc(ndims, sizeof(MPI_Offset)); - stride_copy = (MPI_Offset *)calloc(ndims, sizeof(MPI_Offset)); - - /* copy values */ - if (startp) { - copy(startp, startp+ndims, start_copy); - } - else { - /* default to 0,0,...,0 */ - fill(start_copy, start_copy+ndims, 0); - } - - /* set counts */ - if (countp) { - copy(countp, countp+ndims, count_copy); - } - else { - /* defaults to dimlen[0], dimlen[1], ... dimlen[ndims-1] */ - - dimids = (int *) calloc(ndims, sizeof(int)); - - /* get the diminsion IDs for this variable */ - rc = ncmpi_inq_vardimid(ncid, varid, dimids); - if (rc != NC_NOERR) { - log_error(debug_level, "could not get dimids"); - goto cleanup; - } - - /* set count[i] to dimlen[i] */ - for (i = 0; incid = ncid; - chunk->varid = varid; - chunk->ndims = ndims; - chunk->buf = buf; - chunk->atype = atype; - chunk->len = len; - chunk->datatype = datatype; - chunk->num_elements = count; - chunk->start = start_copy; - chunk->count = count_copy; - chunk->stride = stride_copy; - add_participant_chunk(caller, chunk); - - rc = NSSI_OK; - - if (aggregate_data_ready_to_write(ncid, varid)) { - log_debug(netcdf_debug_level, "ready to agg"); - int chunk_count=0; - - trios_declare_timer(aggTime); - trios_start_timer(aggTime); - while(try_aggregation(ncid, varid) == TRUE); - trios_stop_timer("agg", aggTime); - - aggregation_chunk_details_t **chunks = get_chunks(ncid, varid, &chunk_count); - if (chunk_count == 0) { - log_error(netcdf_debug_level, "chunk_count is 0. how can that be?"); - goto cleanup; - } - - if (use_collective(ncid)) { - log_debug(netcdf_debug_level, "using collective"); - trios_declare_timer(collPutTime); - trios_start_timer(collPutTime); - rc = do_put_vars_all(ncid, varid, chunks, chunk_count); - trios_stop_timer("coll put", collPutTime); - if (rc != NC_NOERR) { - log_error(debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - free(chunks); - goto cleanup; - } - } else { - log_debug(netcdf_debug_level, "using independent"); - trios_declare_timer(indepPutTime); - trios_start_timer(indepPutTime); - rc = do_put_vars(chunks, chunk_count); - trios_stop_timer("indep put", indepPutTime); - if (rc != NC_NOERR) { - log_error(debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - free(chunks); - goto cleanup; - } - } - - free(chunks); - - cleanup_aggregation_chunks(ncid, varid); - } else { - log_debug(netcdf_debug_level, "not ready to agg"); - } - } else if (use_caching(ncid)) { - log_debug(netcdf_debug_level, "using caching"); - aggregation_chunk_details_t *chunk; - chunk = new aggregation_chunk_details_t; - chunk->ncid = ncid; - chunk->varid = varid; - chunk->ndims = ndims; - chunk->buf = buf; - chunk->atype = atype; - chunk->len = len; - chunk->datatype = datatype; - chunk->num_elements = count; - chunk->start = start_copy; - chunk->count = count_copy; - chunk->stride = stride_copy; - add_participant_chunk(caller, chunk); - - rc = NSSI_OK; - - if (cache_data_ready_to_write(ncid, varid)) { - log_debug(netcdf_debug_level, "ready to flush cache"); - int chunk_count=0; - - aggregation_chunk_details_t **chunks = get_chunks(ncid, varid, &chunk_count); - if (chunk_count == 0) { - log_error(netcdf_debug_level, "chunk_count is 0. how can that be?"); - goto cleanup; - } - - if (use_collective(ncid)) { - log_debug(netcdf_debug_level, "using collective"); - trios_declare_timer(collPutTime); - trios_start_timer(collPutTime); - rc = do_put_vars_all(ncid, varid, chunks, chunk_count); - trios_stop_timer("coll put", collPutTime); - if (rc != NC_NOERR) { - log_error(debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - free(chunks); - goto cleanup; - } - } else { - log_debug(netcdf_debug_level, "using independent"); - trios_declare_timer(indepPutTime); - trios_start_timer(indepPutTime); - rc = do_put_vars(chunks, chunk_count); - trios_stop_timer("indep put", indepPutTime); - if (rc != NC_NOERR) { - log_error(debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - free(chunks); - goto cleanup; - } - } - - free(chunks); - - cleanup_aggregation_chunks(ncid, varid); - } else { - log_debug(netcdf_debug_level, "not ready to flush cache"); - } - } else { - log_debug(netcdf_debug_level, "using direct"); - rc = do_put_vars(ncid, varid, start_copy, count_copy, stride_copy, - buf, count, datatype); - if (rc != NC_NOERR) { - log_error(debug_level, "Failed to put variable: %s", ncmpi_strerror(rc)); - goto cleanup; - } - } - } - -cleanup: - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - if (dimids) free(dimids); - - /* TODO: add the job to the job queue */ - if ((!use_aggregation(ncid)) && - (!use_caching(ncid))) { - if (count_copy) free(count_copy); - if (start_copy) free(start_copy); - if (stride_copy) free(stride_copy); - free(buf); - } - - trios_stop_timer("put var", callTime); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -/** - * Read a variable to the underlying netcdf file. - * - * This implementation will push raw data to the client - * using the data address. - * - * TODO: Put data in a cache before calling the actual netcdf function. - * - */ -int nc_get_vars_stub( - const unsigned long request_id, - const NNTI_peer_t *caller, - const nc_get_vars_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NC_NOERR; - int i, ndims; - - const int ncid = args->ncid; - const int varid = args->varid; - const int atype = args->atype; - const nc_size_t *stridep = args->stride.stride_val; - const nc_size_t *startp = args->start.start_val; - const nc_size_t *countp = args->count.count_val; - const ssize_t len = args->len; - - MPI_Offset *count_copy = NULL; - MPI_Offset *start_copy = NULL; - MPI_Offset *stride_copy = NULL; - int *dimids = NULL; - - nc_type vartype; - log_level debug_level = netcdf_debug_level; - - MPI_Datatype datatype; - void *buf = malloc(len); - - - log_debug(netcdf_debug_level, "enter"); - - log_debug(debug_level, "calling nc_get_vars_stub(ncid=%d, varid=%d, atype=%d, len=%d)", ncid, varid, atype, len); - - trios_declare_timer(callTime); - trios_start_timer(callTime); - if (use_aggregation(ncid)) { - flush_aggregated_chunks(ncid, varid); - } else if (use_caching(ncid)) { - flush_cached_chunks(ncid, varid); - } - trios_stop_timer("flush on sync", callTime); - - - rc = ncmpi_inq_varndims(ncid, varid, &ndims); - if (rc != NC_NOERR) { - log_error(debug_level, "Could not get ndims: %s", ncmpi_strerror(rc)); - goto cleanup; - } - - if (ndims) { - size_t count = 1, ccount=0; - - /* find out the right type for this variable */ - rc = ncmpi_inq_vartype(ncid, varid, &vartype); - if (rc != NC_NOERR) { - log_error(debug_level, "Unable to get variable type"); - goto cleanup; - } - - /* Treat each arg type differently */ - switch (vartype) { - - case NC_BYTE: - datatype = MPI_BYTE; - count = len; - log_debug(debug_level, "using MPI_BYTE"); - break; - - case NC_CHAR: - datatype = MPI_CHAR; - count = len/sizeof(char); - log_debug(debug_level, "using MPI_CHAR"); - break; - - case NC_SHORT: - datatype = MPI_SHORT; - count = len/sizeof(short); - log_debug(debug_level, "using MPI_SHORT"); - break; - - case NC_INT: - datatype = MPI_INT; - count = len/sizeof(int); - log_debug(debug_level, "using MPI_INT"); - break; - - case NC_FLOAT: - datatype = MPI_FLOAT; - count = len/sizeof(float); - log_debug(debug_level, "using MPI_FLOAT"); - break; - - case NC_DOUBLE: - datatype = MPI_DOUBLE; - count = len/sizeof(double); - log_debug(debug_level, "using MPI_DOUBLE"); - break; - - default: - log_error(debug_level, "Operation=%d not supported", vartype); - rc = NC_ENOTSUPP; - goto cleanup; - } - - log_debug(debug_level, "Sending %d vals (%lu bytes) to client", count, len); - - start_copy = (MPI_Offset *)calloc(ndims, sizeof(MPI_Offset)); - count_copy = (MPI_Offset *)calloc(ndims, sizeof(MPI_Offset)); - stride_copy = (MPI_Offset *)calloc(ndims, sizeof(MPI_Offset)); - - /* copy values */ - if (startp) { - copy(startp, startp+ndims, start_copy); - } - else { - /* default to 0,0,...,0 */ - fill(start_copy, start_copy+ndims, 0); - } - - /* set counts */ - if (countp) { - copy(countp, countp+ndims, count_copy); - } - else { - /* defaults to dimlen[0], dimlen[1], ... dimlen[ndims-1] */ - - dimids = (int *) calloc(ndims, sizeof(int)); - - /* get the diminsion IDs for this variable */ - rc = ncmpi_inq_vardimid(ncid, varid, dimids); - if (rc != NC_NOERR) { - log_error(debug_level, "could not get dimids"); - goto cleanup; - } - - /* set count[i] to dimlen[i] */ - for (i = 0; incid; - const int new_mode = args->new_fill_mode; - nc_set_fill_res res; /* this is what we send back to the client */ - - log_debug(netcdf_debug_level, "enter"); - - rc=ncmpi_set_fill(ncid, new_mode, &res.old_fill_mode); - - rc = nssi_send_result(caller, request_id, rc, &res, res_addr); - - log_debug(netcdf_debug_level, "exit"); - - return rc; -} - -/* -------- END SERVER-SIDE STUBS -------------- */ - -int netcdf_server_init() -{ - /* register server stubs */ - NSSI_REGISTER_SERVER_STUB(NETCDF_CREATE_OP, nc_create_stub, nc_create_args, nc_create_res); - NSSI_REGISTER_SERVER_STUB(NETCDF_OPEN_OP, nc_open_stub, nc_open_args, nc_open_res); - NSSI_REGISTER_SERVER_STUB(NETCDF_DEF_DIM_OP, nc_def_dim_stub, nc_def_dim_args, int); - NSSI_REGISTER_SERVER_STUB(NETCDF_DEF_VAR_OP, nc_def_var_stub, nc_def_var_args, int); - NSSI_REGISTER_SERVER_STUB(NETCDF_GET_ATT_OP, nc_get_att_stub, nc_get_att_args, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_PUT_ATT_OP, nc_put_att_stub, nc_put_att_args, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_REDEF_OP, nc_redef_stub, int, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_ENDDEF_OP, nc_enddef_stub, int, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_PUT_VARS_OP, nc_put_vars_stub, nc_put_vars_args, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_GET_VARS_OP, nc_get_vars_stub, nc_get_vars_args, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_SYNC_OP, nc_sync_stub, int, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_CLOSE_OP, nc_close_stub, int, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_BEGIN_INDEP_OP, nc_begin_indep_stub, int, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_END_INDEP_OP, nc_end_indep_stub, int, void); - NSSI_REGISTER_SERVER_STUB(NETCDF_SET_FILL_OP, nc_set_fill_stub, nc_set_fill_args, nc_set_fill_res); - - return 0; -} - - - -/* -- private methods -- */ -static void print_opts( - FILE *fp, - const struct netcdf_args *args_info, - const char *prefix) -{ - fprintf(fp, "%s ------------ NETCDF SERVER ---------\n", prefix); - - fprintf(fp, "%s ------------ ARGUMENTS -----------\n", prefix); - fprintf(fp, "%s \tserver-url = %s\n", prefix, args_info->server_url.c_str()); - fprintf(fp, "%s \tserver-url-file = %s\n", prefix, args_info->url_file.c_str()); - fprintf(fp, "%s \tdaemon = %s\n", prefix, (args_info->daemon_flag)?"true":"false"); - fprintf(fp, "%s \tverbose = %d\n", prefix, args_info->verbose); - fprintf(fp, "%s \tlogfile = %s\n", prefix, args_info->logfile.c_str()); - - fprintf(fp, "%s -----------------------------------\n", prefix); -} - -static void generate_contact_info(char *myid) -{ - NNTI_url *all_urls=NULL; - int rank, np; - char contact_path[1024]; - log_level debug_level = netcdf_debug_level; - debug_level = LOG_ALL; - - log_debug(netcdf_debug_level, "enter"); - - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - log_debug(debug_level, "rank (%d)", rank); - - if (rank==0) { - MPI_Comm_size(MPI_COMM_WORLD, &np); - all_urls=(NNTI_url *)malloc(np*sizeof(NNTI_url)); - } - MPI_Gather(myid, sizeof(NNTI_url), MPI_BYTE, - all_urls, sizeof(NNTI_url), MPI_BYTE, - 0, MPI_COMM_WORLD); - if (rank==0) { - char *contact_file=getenv("NETCDF_CONTACT_INFO"); - if (contact_file==NULL) { - log_error(debug_level, "NETCDF_CONTACT_INFO env var is undefined."); - free(all_urls); - return; - } - sprintf(contact_path, "%s.tmp", contact_file); - log_debug(debug_level, "creating contact file (%s)", contact_path); - FILE *f=fopen(contact_path, "w"); - if (f==NULL) { - perror("fopen"); - } - for (int i=0;i - - - - - - - - - - - - - - - - diff --git a/packages/trios/services/netcdf/tests/CMakeLists.txt b/packages/trios/services/netcdf/tests/CMakeLists.txt deleted file mode 100644 index c6d2ab8337fb..000000000000 --- a/packages/trios/services/netcdf/tests/CMakeLists.txt +++ /dev/null @@ -1,316 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -# Need to include the binary dir so we can find config.h -INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}) - -# Need to include the source dir so we can find aggregation.h -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src) -# Need to include the binary dir so we can find netcdf_args.h -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/../src) - -LINK_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/../src) - -cmake_policy(VERSION 2.8) - - -###################### UNIT TESTS ################################################## - -TRIBITS_ADD_EXECUTABLE( - netcdf-remote-test - SOURCES netcdf_test.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - netcdf-test - SOURCES netcdf_test.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - aggregation_test - SOURCES aggregation_test.cpp ../src/aggregation.cpp ../src/netcdf_debug.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - - -###################### PROGRAMS ################################################## - -TRIBITS_ADD_EXECUTABLE( - simple_xy_wr - SOURCES simple_xy_wr.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - simple_xy_wr_remote - SOURCES simple_xy_wr.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - simple_xy_rd - SOURCES simple_xy_rd.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - simple_xy_rd_remote - SOURCES simple_xy_rd.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_xy_wr - SOURCES ncmpi_xy_wr.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_xy_wr_remote - SOURCES ncmpi_xy_wr.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_xy_rd - SOURCES ncmpi_xy_rd.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_xy_rd_remote - SOURCES ncmpi_xy_rd.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - - -######### PRES_TEMP_4D EXAMPLE #################### - -TRIBITS_ADD_EXECUTABLE( - pres_temp_4D_wr - SOURCES pres_temp_4D_wr.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - pres_temp_4D_wr_remote - SOURCES pres_temp_4D_wr.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - pres_temp_4D_rd - SOURCES pres_temp_4D_rd.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - pres_temp_4D_rd_remote - SOURCES pres_temp_4D_rd.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_pres_temp_4D_wr - SOURCES ncmpi_pres_temp_4D_wr.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_pres_temp_4D_wr_remote - SOURCES ncmpi_pres_temp_4D_wr.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_pres_temp_4D_rd - SOURCES ncmpi_pres_temp_4D_rd.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - ncmpi_pres_temp_4D_rd_remote - SOURCES ncmpi_pres_temp_4D_rd.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - - -######### SVC_PRES_TEMP EXAMPLE #################### - -TRIBITS_ADD_EXECUTABLE( - sfc_pres_temp_wr - SOURCES sfc_pres_temp_wr.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - sfc_pres_temp_wr_remote - SOURCES sfc_pres_temp_wr.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - sfc_pres_temp_rd - SOURCES sfc_pres_temp_rd.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - sfc_pres_temp_rd_remote - SOURCES sfc_pres_temp_rd.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -######### Integer tests from parallel-netcdf distribution #################### - -TRIBITS_ADD_EXECUTABLE( - test_read_coll - SOURCES test_read_coll.c testutils.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_read_coll_remote - SOURCES test_read_coll.c testutils.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_read_indep - SOURCES test_read_indep.c testutils.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_read_indep_remote - SOURCES test_read_indep.c testutils.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_write_coll - SOURCES test_write_coll.c testutils.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_write_coll_remote - SOURCES test_write_coll.c testutils.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_write_indep - SOURCES test_write_indep.c testutils.c - DEPLIBS ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - test_write_indep_remote - SOURCES test_write_indep.c testutils.c - DEPLIBS ${Trios_NETCDF_LIBRARIES} ${DEPLIBS} - COMM serial mpi - NOEXEPREFIX -) diff --git a/packages/trios/services/netcdf/tests/aggregation_test.cpp b/packages/trios/services/netcdf/tests/aggregation_test.cpp deleted file mode 100644 index cec05b9e9b57..000000000000 --- a/packages/trios/services/netcdf/tests/aggregation_test.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include -#include -#include -#include -#include - -#include "Trios_nssi_client.h" -#include "netcdf_debug.h" - -#include "aggregation.h" - -int ncid=0; /* netCDF id */ -NNTI_peer_t caller; - -int varid=1; -int ndims=4; - -int dims[4] = {3, 3, 3, 3}; -MPI_Offset *new_chunk_start=NULL; -MPI_Offset *new_chunk_count=NULL; -MPI_Offset *new_chunk_stride=NULL; - -void add_chunk(int *buf) -{ - int buf_len; - int num_elements; - aggregation_chunk_details_t *chunk; - - new_chunk_count=(MPI_Offset *)malloc(ndims*sizeof(MPI_Offset)); - new_chunk_stride=(MPI_Offset *)malloc(ndims*sizeof(MPI_Offset)); - - new_chunk_count[0]=dims[0];new_chunk_count[1]=dims[1];new_chunk_count[2]=dims[2];new_chunk_count[3]=dims[3]; - new_chunk_stride[0]=1;new_chunk_stride[1]=1;new_chunk_stride[2]=1;new_chunk_stride[3]=1; - buf_len=dims[0]*dims[1]*dims[2]*dims[3]*sizeof(int); - num_elements=dims[0]*dims[1]*dims[2]*dims[3]; - - - add_participant_for_file(ncid, - &caller, - WRITE_AGGREGATE_INDEPENDENT); - - chunk = new aggregation_chunk_details_t; - chunk->ncid = ncid; - chunk->varid = varid; - chunk->ndims = ndims; - - chunk->buf = buf; - - chunk->atype = NC_INT; - chunk->len = buf_len; - - chunk->datatype = MPI_INT; - chunk->num_elements = num_elements; - - chunk->start = new_chunk_start; - chunk->count = new_chunk_count; - chunk->stride = new_chunk_stride; - - add_participant_chunk(&caller, chunk); - -} - -/* - * NB: The dims of the individual chunks is the same as the dimensions - * of the overall array in terms of chunks. So if the dimensions of a chunk - * is 3x3x3, then the overall array will have 3x3x3 chunks or 9x9x9 elements. - */ -int -main(int argc, char **argv) -{ - int test_chunk_count; - int **chunk; - - int i=0,j=0,k=0,l=0; - int buf_len; - - MPI_Init(&argc, &argv); - - netcdf_debug_level=LOG_ALL; - /* initialize and enable logging */ - logger_init(netcdf_debug_level, NULL); - - test_chunk_count=1; - buf_len=1; - for (i=0;i -#include -#include -#include - -/* This is the name of the data file we will read. */ -#define FILE_NAME "pres_temp_4D.nc" - -/* We are reading 4D data, a 2 x 6 x 12 lvl-lat-lon grid, with 2 - timesteps of data. */ -#define NDIMS 4 -#define NLAT 6 -#define NLON 12 -#define LAT_NAME "latitude" -#define LON_NAME "longitude" -#define NREC 2 -#define REC_NAME "time" -#define LVL_NAME "level" -#define NLVL 2 - -/* Names of things. */ -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" -#define UNITS "units" -#define DEGREES_EAST "degrees_east" -#define DEGREES_NORTH "degrees_north" - -/* These are used to calculate the values we expect to find. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* For the units attributes. */ -#define UNITS "units" -#define PRES_UNITS "hPa" -#define TEMP_UNITS "celsius" -#define LAT_UNITS "degrees_north" -#define LON_UNITS "degrees_east" -#define MAX_ATT_LEN 80 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERR(e) {printf("Error: %s\n", ncmpi_strerror(e)); return 2;} - -int -main(int argc, char *argv[]) -{ - int ncid, pres_varid, temp_varid; - int lat_varid, lon_varid; - - MPI_Init(&argc, &argv); - - /* The start and count arrays will tell the netCDF library where to - read our data. */ - MPI_Offset start[NDIMS], count[NDIMS]; - - /* Program variables to hold the data we will read. We will only - need enough space to hold one timestep of data; one record. */ - float pres_in[NLVL][NLAT][NLON]; - float temp_in[NLVL][NLAT][NLON]; - - /* These program variables hold the latitudes and longitudes. */ - float lats[NLAT], lons[NLON]; - - /* Loop indexes. */ - int lvl, lat, lon, rec, i = 0; - - /* Error handling. */ - int retval; - - /* Open the file. */ - if ((retval = ncmpi_open(MPI_COMM_WORLD, FILE_NAME, NC_NOWRITE, MPI_INFO_NULL, &ncid))) - ERR(retval); - - /* Get the varids of the latitude and longitude coordinate - * variables. */ - if ((retval = ncmpi_inq_varid(ncid, LAT_NAME, &lat_varid))) - ERR(retval); - if ((retval = ncmpi_inq_varid(ncid, LON_NAME, &lon_varid))) - ERR(retval); - - /* Read the coordinate variable data. */ - if ((retval = ncmpi_begin_indep_data(ncid))) - ERR(retval); - if ((retval = ncmpi_get_var_float(ncid, lat_varid, &lats[0]))) - ERR(retval); - if ((retval = ncmpi_get_var_float(ncid, lon_varid, &lons[0]))) - ERR(retval); - if ((retval = ncmpi_end_indep_data(ncid))) - ERR(retval); - - /* Check the coordinate variable data. */ - for (lat = 0; lat < NLAT; lat++) - if (lats[lat] != START_LAT + 5.*lat) - return 2; - for (lon = 0; lon < NLON; lon++) - if (lons[lon] != START_LON + 5.*lon) - return 2; - - /* Get the varids of the pressure and temperature netCDF - * variables. */ - if ((retval = ncmpi_inq_varid(ncid, PRES_NAME, &pres_varid))) - ERR(retval); - if ((retval = ncmpi_inq_varid(ncid, TEMP_NAME, &temp_varid))) - ERR(retval); - - /* Read the data. Since we know the contents of the file we know - * that the data arrays in this program are the correct size to - * hold one timestep. */ - count[0] = 1; - count[1] = NLVL; - count[2] = NLAT; - count[3] = NLON; - start[1] = 0; - start[2] = 0; - start[3] = 0; - - /* Read and check one record at a time. */ - for (rec = 0; rec < NREC; rec++) - { - start[0] = rec; - if ((retval = ncmpi_begin_indep_data(ncid))) - ERR(retval); - if ((retval = ncmpi_get_vara_float(ncid, pres_varid, start, - count, &pres_in[0][0][0]))) - ERR(retval); - if ((retval = ncmpi_get_vara_float(ncid, temp_varid, start, - count, &temp_in[0][0][0]))) - ERR(retval); - if ((retval = ncmpi_end_indep_data(ncid))) - ERR(retval); - - /* Check the data. */ - i = 0; - for (lvl = 0; lvl < NLVL; lvl++) - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) - { - if (pres_in[lvl][lat][lon] != SAMPLE_PRESSURE + i) - return 2; - if (temp_in[lvl][lat][lon] != SAMPLE_TEMP + i) - return 2; - i++; - } - - } /* next record */ - - /* Close the file. */ - if ((retval = ncmpi_close(ncid))) - ERR(retval); - - printf("*** SUCCESS reading example file pres_temp_4D.nc!\n"); - - MPI_Finalize(); - - return 0; -} diff --git a/packages/trios/services/netcdf/tests/ncmpi_pres_temp_4D_wr.c b/packages/trios/services/netcdf/tests/ncmpi_pres_temp_4D_wr.c deleted file mode 100644 index ddb81e70b149..000000000000 --- a/packages/trios/services/netcdf/tests/ncmpi_pres_temp_4D_wr.c +++ /dev/null @@ -1,231 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This is an example program which writes some 4D pressure and - temperatures. It is intended to illustrate the use of the netCDF - C API. The companion program pres_temp_4D_rd.c shows how - to read the netCDF data file created by this program. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: pres_temp_4D_wr.c,v 1.4 2006/06/29 20:45:58 russ Exp $ - */ - -#include -#include -#include -#include - -/* This is the name of the data file we will create. */ -#define FILE_NAME "pres_temp_4D.nc" - -/* We are writing 4D data, a 2 x 6 x 12 lvl-lat-lon grid, with 2 - timesteps of data. */ -#define NDIMS 4 -#define NLAT 6 -#define NLON 12 -#define LAT_NAME "latitude" -#define LON_NAME "longitude" -#define NREC 2 -#define REC_NAME "time" -#define LVL_NAME "level" -#define NLVL 2 - -/* Names of things. */ -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" -#define UNITS "units" -#define DEGREES_EAST "degrees_east" -#define DEGREES_NORTH "degrees_north" - -/* These are used to construct some example data. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* For the units attributes. */ -#define UNITS "units" -#define PRES_UNITS "hPa" -#define TEMP_UNITS "celsius" -#define LAT_UNITS "degrees_north" -#define LON_UNITS "degrees_east" -#define MAX_ATT_LEN 80 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERR(e) {printf("Error: %s\n", ncmpi_strerror(e)); return 2;} - -int -main(int argc, char *argv[]) -{ - /* IDs for the netCDF file, dimensions, and variables. */ - int ncid, lon_dimid, lat_dimid, lvl_dimid, rec_dimid; - int lat_varid, lon_varid, pres_varid, temp_varid; - int dimids[NDIMS]; - - /* The start and count arrays will tell the netCDF library where to - write our data. */ - MPI_Offset start[NDIMS], count[NDIMS]; - - /* Program variables to hold the data we will write out. We will only - need enough space to hold one timestep of data; one record. */ - float pres_out[NLVL][NLAT][NLON]; - float temp_out[NLVL][NLAT][NLON]; - - /* These program variables hold the latitudes and longitudes. */ - float lats[NLAT], lons[NLON]; - - /* Loop indexes. */ - int lvl, lat, lon, rec, i = 0; - - /* Error handling. */ - int retval; - - MPI_Init(&argc, &argv); - - /* Create some pretend data. If this wasn't an example program, we - * would have some real data to write, for example, model - * output. */ - for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5.*lat; - for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5.*lon; - - for (lvl = 0; lvl < NLVL; lvl++) - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) - { - pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + i; - temp_out[lvl][lat][lon] = SAMPLE_TEMP + i++; - } - - /* Create the file. */ - if ((retval = ncmpi_create(MPI_COMM_WORLD, FILE_NAME, NC_CLOBBER, MPI_INFO_NULL, &ncid))) - ERR(retval); - - /* Define the dimensions. The record dimension is defined to have - * unlimited length - it can grow as needed. In this example it is - * the time dimension.*/ - if ((retval = ncmpi_def_dim(ncid, LVL_NAME, NLVL, &lvl_dimid))) - ERR(retval); - if ((retval = ncmpi_def_dim(ncid, LAT_NAME, NLAT, &lat_dimid))) - ERR(retval); - if ((retval = ncmpi_def_dim(ncid, LON_NAME, NLON, &lon_dimid))) - ERR(retval); - if ((retval = ncmpi_def_dim(ncid, REC_NAME, NC_UNLIMITED, &rec_dimid))) - ERR(retval); - - /* Define the coordinate variables. We will only define coordinate - variables for lat and lon. Ordinarily we would need to provide - an array of dimension IDs for each variable's dimensions, but - since coordinate variables only have one dimension, we can - simply provide the address of that dimension ID (&lat_dimid) and - similarly for (&lon_dimid). */ - if ((retval = ncmpi_def_var(ncid, LAT_NAME, NC_FLOAT, 1, &lat_dimid, - &lat_varid))) - ERR(retval); - if ((retval = ncmpi_def_var(ncid, LON_NAME, NC_FLOAT, 1, &lon_dimid, - &lon_varid))) - ERR(retval); - - /* Assign units attributes to coordinate variables. */ - if ((retval = ncmpi_put_att_text(ncid, lat_varid, UNITS, - strlen(DEGREES_NORTH), DEGREES_NORTH))) - ERR(retval); - if ((retval = ncmpi_put_att_text(ncid, lon_varid, UNITS, - strlen(DEGREES_EAST), DEGREES_EAST))) - ERR(retval); - - /* The dimids array is used to pass the dimids of the dimensions of - the netCDF variables. Both of the netCDF variables we are - creating share the same four dimensions. In C, the - unlimited dimension must come first on the list of dimids. */ - dimids[0] = rec_dimid; - dimids[1] = lvl_dimid; - dimids[2] = lat_dimid; - dimids[3] = lon_dimid; - - /* Define the netCDF variables for the pressure and temperature - * data. */ - if ((retval = ncmpi_def_var(ncid, PRES_NAME, NC_FLOAT, NDIMS, - dimids, &pres_varid))) - ERR(retval); - if ((retval = ncmpi_def_var(ncid, TEMP_NAME, NC_FLOAT, NDIMS, - dimids, &temp_varid))) - ERR(retval); - - /* Assign units attributes to the netCDF variables. */ - if ((retval = ncmpi_put_att_text(ncid, pres_varid, UNITS, - strlen(PRES_UNITS), PRES_UNITS))) - ERR(retval); - if ((retval = ncmpi_put_att_text(ncid, temp_varid, UNITS, - strlen(TEMP_UNITS), TEMP_UNITS))) - ERR(retval); - - /* End define mode. */ - if ((retval = ncmpi_enddef(ncid))) - ERR(retval); - - /* Write the coordinate variable data. This will put the latitudes - and longitudes of our data grid into the netCDF file. */ - if ((retval = ncmpi_begin_indep_data(ncid))) - ERR(retval); - - if ((retval = ncmpi_put_var_float(ncid, lat_varid, &lats[0]))) - ERR(retval); - - if ((retval = ncmpi_put_var_float(ncid, lon_varid, &lons[0]))) - ERR(retval); - - if ((retval = ncmpi_end_indep_data(ncid))) - ERR(retval); - - /* These settings tell netcdf to write one timestep of data. (The - setting of start[0] inside the loop below tells netCDF which - timestep to write.) */ - count[0] = 1; - count[1] = NLVL; - count[2] = NLAT; - count[3] = NLON; - start[1] = 0; - start[2] = 0; - start[3] = 0; - - /* Write the pretend data. This will write our surface pressure and - surface temperature data. The arrays only hold one timestep worth - of data. We will just rewrite the same data for each timestep. In - a real application, the data would change between timesteps. */ - for (rec = 0; rec < NREC; rec++) - { - start[0] = rec; - - if ((retval = ncmpi_begin_indep_data(ncid))) - ERR(retval); - - if ((retval = ncmpi_put_vara_float(ncid, pres_varid, start, count, - &pres_out[0][0][0]))) - ERR(retval); - if ((retval = ncmpi_put_vara_float(ncid, temp_varid, start, count, - &temp_out[0][0][0]))) - ERR(retval); - - if ((retval = ncmpi_end_indep_data(ncid))) - ERR(retval); - } - - /* Close the file. */ - if ((retval = ncmpi_close(ncid))) - ERR(retval); - - printf("*** SUCCESS writing example file %s!\n", FILE_NAME); - - MPI_Finalize(); - - return 0; -} diff --git a/packages/trios/services/netcdf/tests/ncmpi_test.c b/packages/trios/services/netcdf/tests/ncmpi_test.c deleted file mode 100644 index c4b221b80e33..000000000000 --- a/packages/trios/services/netcdf/tests/ncmpi_test.c +++ /dev/null @@ -1,157 +0,0 @@ -#include -#include -#include -#include - -void -check_err(const int stat, const int line, const char *file) { - if (stat != NC_NOERR) { - (void) fprintf(stderr, "line %d of %s: %s\n", line, file, ncmpi_strerror(stat)); - exit(1); - } -} - -int -main(int argc, char **argv) { /* create foo.nc */ - - int stat; /* return status */ - int ncid; /* netCDF id */ - - /* dimension ids */ - int lat_dim; - int lon_dim; - int time_dim; - - /* dimension lengths */ - size_t lat_len = 10; - size_t lon_len = 5; - size_t time_len = NC_UNLIMITED; - - /* variable ids */ - int lat_id; - int lon_id; - int time_id; - int z_id; - int t_id; - int p_id; - int rh_id; - - /* rank (number of dimensions) for each variable */ -# define RANK_lat 1 -# define RANK_lon 1 -# define RANK_time 1 -# define RANK_z 3 -# define RANK_t 3 -# define RANK_p 3 -# define RANK_rh 3 - - /* variable shapes */ - int lat_dims[RANK_lat]; - int lon_dims[RANK_lon]; - int time_dims[RANK_time]; - int z_dims[RANK_z]; - int t_dims[RANK_t]; - int p_dims[RANK_p]; - int rh_dims[RANK_rh]; - - /* attribute vectors */ - double z_valid_range[2]; - double p__FillValue[1]; - int rh__FillValue[1]; - - int stat=0; - MPI_Init(&argc, &argv); - /* enter define mode */ - stat = ncmpi_create(MPI_COMM_WORLD, "foo.nc", NC_CLOBBER, MPI_INFO_NULL, &ncid); - check_err(stat,__LINE__,__FILE__); - - /* define dimensions */ - stat = ncmpi_def_dim(ncid, "lat", lat_len, &lat_dim); - check_err(stat,__LINE__,__FILE__); - stat = ncmpi_def_dim(ncid, "lon", lon_len, &lon_dim); - check_err(stat,__LINE__,__FILE__); - stat = ncmpi_def_dim(ncid, "time", time_len, &time_dim); - check_err(stat,__LINE__,__FILE__); - - /* define variables */ - - lat_dims[0] = lat_dim; - stat = ncmpi_def_var(ncid, "lat", NC_INT, RANK_lat, lat_dims, &lat_id); - check_err(stat,__LINE__,__FILE__); - - lon_dims[0] = lon_dim; - stat = ncmpi_def_var(ncid, "lon", NC_INT, RANK_lon, lon_dims, &lon_id); - check_err(stat,__LINE__,__FILE__); - - time_dims[0] = time_dim; - stat = ncmpi_def_var(ncid, "time", NC_INT, RANK_time, time_dims, &time_id); - check_err(stat,__LINE__,__FILE__); - - z_dims[0] = time_dim; - z_dims[1] = lat_dim; - z_dims[2] = lon_dim; - stat = ncmpi_def_var(ncid, "z", NC_FLOAT, RANK_z, z_dims, &z_id); - check_err(stat,__LINE__,__FILE__); - - t_dims[0] = time_dim; - t_dims[1] = lat_dim; - t_dims[2] = lon_dim; - stat = ncmpi_def_var(ncid, "t", NC_FLOAT, RANK_t, t_dims, &t_id); - check_err(stat,__LINE__,__FILE__); - - p_dims[0] = time_dim; - p_dims[1] = lat_dim; - p_dims[2] = lon_dim; - stat = ncmpi_def_var(ncid, "p", NC_DOUBLE, RANK_p, p_dims, &p_id); - check_err(stat,__LINE__,__FILE__); - - rh_dims[0] = time_dim; - rh_dims[1] = lat_dim; - rh_dims[2] = lon_dim; - stat = ncmpi_def_var(ncid, "rh", NC_INT, RANK_rh, rh_dims, &rh_id); - check_err(stat,__LINE__,__FILE__); - - /* assign attributes */ - stat = ncmpi_put_att_text(ncid, lat_id, "units", 13, "degrees_north"); - check_err(stat,__LINE__,__FILE__); - stat = ncmpi_put_att_text(ncid, lon_id, "units", 12, "degrees_east"); - check_err(stat,__LINE__,__FILE__); - stat = ncmpi_put_att_text(ncid, time_id, "units", 7, "seconds"); - check_err(stat,__LINE__,__FILE__); - stat = ncmpi_put_att_text(ncid, z_id, "units", 6, "meters"); - check_err(stat,__LINE__,__FILE__); - z_valid_range[0] = 0; - z_valid_range[1] = 5000; - stat = ncmpi_put_att_double(ncid, z_id, "valid_range", NC_DOUBLE, 2, z_valid_range); - check_err(stat,__LINE__,__FILE__); - p__FillValue[0] = -9999; - stat = ncmpi_put_att_double(ncid, p_id, "_FillValue", NC_DOUBLE, 1, p__FillValue); - check_err(stat,__LINE__,__FILE__); - rh__FillValue[0] = -1; - stat = ncmpi_put_att_int(ncid, rh_id, "_FillValue", NC_INT, 1, rh__FillValue); - check_err(stat,__LINE__,__FILE__); - - /* leave define mode */ - stat = ncmpi_enddef (ncid); - check_err(stat,__LINE__,__FILE__); - - { /* store lat */ - static int lat[] = {0, 10, 20, 30, 40, 50, 60, 70, 80, 90}; - ncmpi_begin_indep_data(ncid); - stat = ncmpi_put_var_int(ncid, lat_id, lat); - ncmpi_end_indep_data(ncid); - check_err(stat,__LINE__,__FILE__); - } - - { /* store lon */ - static int lon[] = {-140, -118, -96, -84, -52}; - ncmpi_begin_indep_data(ncid); - stat = ncmpi_put_var_int(ncid, lon_id, lon); - ncmpi_end_indep_data(ncid); - check_err(stat,__LINE__,__FILE__); - } - stat = ncmpi_close(ncid); - check_err(stat,__LINE__,__FILE__); - MPI_Finalize(); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/ncmpi_xy_rd.c b/packages/trios/services/netcdf/tests/ncmpi_xy_rd.c deleted file mode 100644 index 2368ca828c65..000000000000 --- a/packages/trios/services/netcdf/tests/ncmpi_xy_rd.c +++ /dev/null @@ -1,177 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This is a simple example which reads a small dummy array, which was - written by simple_xy_wr.c. This is intended to illustrate the use - of the netCDF C API. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: simple_xy_rd.c,v 1.9 2006/08/17 23:00:55 russ Exp $ -*/ -#include -#include -#include -#include -#include - -/* This is the name of the data file we will read. */ -#define FILE_NAME "ncmpi_xy.nc" - -/* We are reading 2D data, a 6 x 12 grid. */ -#define NX 6 -#define NY 12 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", ncmpi_strerror(e)); goto cleanup;} - -int -main(int argc, char *argv[]) -{ - /* This will be the netCDF ID for the file and data variable. */ - int ncid, varid; - - int *data_in = NULL; - int data_len[2] = {NX, NY}; - - /* Loop indexes, and error handling. */ - int i, x, y, retval; - int proc_dims[2]; - int proc_periods[2]; - int proc_coord[2]; - int proc_ndims = 2; /* use 2-d topology for processors */ - - /* domain variables */ - MPI_Offset start[2]; - MPI_Offset count[2]; - MPI_Offset stride[2]; - - /* MPI variables and communicators */ - int np, rank; - MPI_Comm comm_cart; - - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &np); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - memset(proc_dims, 0, 2*sizeof(int)); - memset(proc_periods, 0, 2*sizeof(int)); - - memset(start, 0, 2*sizeof(MPI_Offset)); - memset(count, 0, 2*sizeof(MPI_Offset)); - memset(stride, 0, 2*sizeof(MPI_Offset)); - - /* create the dimensions */ - MPI_Dims_create(np, proc_ndims, proc_dims); - - - if (rank == 0) { - fprintf(stdout, "Global Parameters: np=%d, proc_dims = [%d, %d], data_len=[%d, %d]\n\n", - np, proc_dims[0], proc_dims[1], NX, NY); - } - - /* create the cartiesion communicator */ - MPI_Cart_create(MPI_COMM_WORLD, proc_ndims, proc_dims, proc_periods, 0, &comm_cart); - - /* get the coordinates of this process */ - MPI_Cart_coords(comm_cart, rank, 2, proc_coord); - - /* Do the domain decomposition ... assign start, offset, count, stide */ - - /* X dimension */ - for (i=0; i -#include -#include -#include -#include - -/* This is the name of the data file we will create. */ -#define FILE_NAME "ncmpi_xy.nc" - -/* We are writing 2D data, a 6 x 12 grid. */ -#define NDIMS 2 -#define NX 6 -#define NY 12 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", ncmpi_strerror(e)); exit(ERRCODE);} - - -int -main(int argc, char *argv[]) -{ - /* This will be the netCDF ID for the file and data variable. */ - int ncid, varid; - - int *data_out; - int data_len[2] = {NX, NY}; - int dimids[2]; - int x_dimid; - int y_dimid; - int bufcount; - - /* Loop indexes, and error handling. */ - int i, x, y, retval; - int proc_dims[2]; - int proc_periods[2]; - int proc_coord[2]; - int proc_ndims = 2; /* use 2-d topology for processors */ - - /* domain variables */ - MPI_Offset start[2]; - MPI_Offset count[2]; - MPI_Offset stride[2]; - - /* MPI variables and communicators */ - int np, rank; - MPI_Comm comm_cart; - - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &np); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - memset(proc_dims, 0, 2*sizeof(int)); - memset(proc_periods, 0, 2*sizeof(int)); - - memset(start, 0, 2*sizeof(MPI_Offset)); - memset(count, 0, 2*sizeof(MPI_Offset)); - memset(stride, 0, 2*sizeof(MPI_Offset)); - - /* create the dimensions */ - MPI_Dims_create(np, proc_ndims, proc_dims); - - - if (rank == 0) { - fprintf(stdout, "Global Parameters: np=%d, proc_dims = [%d, %d], data_len=[%d, %d]\n\n", - np, proc_dims[0], proc_dims[1], NX, NY); - } - - /* create the cartiesion communicator */ - MPI_Cart_create(MPI_COMM_WORLD, proc_ndims, proc_dims, proc_periods, 0, &comm_cart); - - /* get the coordinates of this process */ - MPI_Cart_coords(comm_cart, rank, 2, proc_coord); - - /* Do the domain decomposition ... assign start, offset, count, stide */ - - /* X dimension */ - for (i=0; i -#include -#include - -void -check_err(const int stat, const int line, const char *file) { - if (stat != NC_NOERR) { - (void) fprintf(stderr, "line %d of %s: %s\n", line, file, nc_strerror(stat)); - exit(1); - } -} - -int -main() { /* create foo.nc */ - - int stat; /* return status */ - int ncid; /* netCDF id */ - - /* dimension ids */ - int lat_dim; - int lon_dim; - int time_dim; - - /* dimension lengths */ - size_t lat_len = 10; - size_t lon_len = 5; - size_t time_len = NC_UNLIMITED; - - /* variable ids */ - int lat_id; - int lon_id; - int time_id; - int z_id; - int t_id; - int p_id; - int rh_id; - - /* rank (number of dimensions) for each variable */ -# define RANK_lat 1 -# define RANK_lon 1 -# define RANK_time 1 -# define RANK_z 3 -# define RANK_t 3 -# define RANK_p 3 -# define RANK_rh 3 - - /* variable shapes */ - int lat_dims[RANK_lat]; - int lon_dims[RANK_lon]; - int time_dims[RANK_time]; - int z_dims[RANK_z]; - int t_dims[RANK_t]; - int p_dims[RANK_p]; - int rh_dims[RANK_rh]; - - /* attribute vectors */ - double z_valid_range[2]; - double p__FillValue[1]; - int rh__FillValue[1]; - - /* enter define mode */ - stat = nc_create("foo.nc", NC_CLOBBER, &ncid); - check_err(stat,__LINE__,__FILE__); - - /* define dimensions */ - stat = nc_def_dim(ncid, "lat", lat_len, &lat_dim); - check_err(stat,__LINE__,__FILE__); - stat = nc_def_dim(ncid, "lon", lon_len, &lon_dim); - check_err(stat,__LINE__,__FILE__); - stat = nc_def_dim(ncid, "time", time_len, &time_dim); - check_err(stat,__LINE__,__FILE__); - - /* define variables */ - - lat_dims[0] = lat_dim; - stat = nc_def_var(ncid, "lat", NC_INT, RANK_lat, lat_dims, &lat_id); - check_err(stat,__LINE__,__FILE__); - - lon_dims[0] = lon_dim; - stat = nc_def_var(ncid, "lon", NC_INT, RANK_lon, lon_dims, &lon_id); - check_err(stat,__LINE__,__FILE__); - - time_dims[0] = time_dim; - stat = nc_def_var(ncid, "time", NC_INT, RANK_time, time_dims, &time_id); - check_err(stat,__LINE__,__FILE__); - - z_dims[0] = time_dim; - z_dims[1] = lat_dim; - z_dims[2] = lon_dim; - stat = nc_def_var(ncid, "z", NC_FLOAT, RANK_z, z_dims, &z_id); - check_err(stat,__LINE__,__FILE__); - - t_dims[0] = time_dim; - t_dims[1] = lat_dim; - t_dims[2] = lon_dim; - stat = nc_def_var(ncid, "t", NC_FLOAT, RANK_t, t_dims, &t_id); - check_err(stat,__LINE__,__FILE__); - - p_dims[0] = time_dim; - p_dims[1] = lat_dim; - p_dims[2] = lon_dim; - stat = nc_def_var(ncid, "p", NC_DOUBLE, RANK_p, p_dims, &p_id); - check_err(stat,__LINE__,__FILE__); - - rh_dims[0] = time_dim; - rh_dims[1] = lat_dim; - rh_dims[2] = lon_dim; - stat = nc_def_var(ncid, "rh", NC_INT, RANK_rh, rh_dims, &rh_id); - check_err(stat,__LINE__,__FILE__); - - /* assign attributes */ - stat = nc_put_att_text(ncid, lat_id, "units", 13, "degrees_north"); - check_err(stat,__LINE__,__FILE__); - stat = nc_put_att_text(ncid, lon_id, "units", 12, "degrees_east"); - check_err(stat,__LINE__,__FILE__); - stat = nc_put_att_text(ncid, time_id, "units", 7, "seconds"); - check_err(stat,__LINE__,__FILE__); - stat = nc_put_att_text(ncid, z_id, "units", 6, "meters"); - check_err(stat,__LINE__,__FILE__); - z_valid_range[0] = 0; - z_valid_range[1] = 5000; - stat = nc_put_att_double(ncid, z_id, "valid_range", NC_DOUBLE, 2, z_valid_range); - check_err(stat,__LINE__,__FILE__); - p__FillValue[0] = -9999; - stat = nc_put_att_double(ncid, p_id, "_FillValue", NC_DOUBLE, 1, p__FillValue); - check_err(stat,__LINE__,__FILE__); - rh__FillValue[0] = -1; - stat = nc_put_att_int(ncid, rh_id, "_FillValue", NC_INT, 1, rh__FillValue); - check_err(stat,__LINE__,__FILE__); - - /* leave define mode */ - stat = nc_enddef (ncid); - check_err(stat,__LINE__,__FILE__); - - { /* store lat */ - static int lat[] = {0, 10, 20, 30, 40, 50, 60, 70, 80, 90}; - stat = nc_put_var_int(ncid, lat_id, lat); - check_err(stat,__LINE__,__FILE__); - } - - { /* store lon */ - static int lon[] = {-140, -118, -96, -84, -52}; - stat = nc_put_var_int(ncid, lon_id, lon); - check_err(stat,__LINE__,__FILE__); - } - stat = nc_close(ncid); - check_err(stat,__LINE__,__FILE__); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/pres_temp_4D_rd.c b/packages/trios/services/netcdf/tests/pres_temp_4D_rd.c deleted file mode 100644 index 21a94404d334..000000000000 --- a/packages/trios/services/netcdf/tests/pres_temp_4D_rd.c +++ /dev/null @@ -1,162 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This is an example which reads some 4D pressure and - temperatures. The data file read by this program is produced by the - companion program pres_temp_4D_wr.c. It is intended to illustrate - the use of the netCDF C API. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: pres_temp_4D_rd.c,v 1.5 2006/06/26 20:37:31 russ Exp $ -*/ - -#include -#include -#include - -/* This is the name of the data file we will read. */ -#define FILE_NAME "pres_temp_4D.nc" - -/* We are reading 4D data, a 2 x 6 x 12 lvl-lat-lon grid, with 2 - timesteps of data. */ -#define NDIMS 4 -#define NLAT 6 -#define NLON 12 -#define LAT_NAME "latitude" -#define LON_NAME "longitude" -#define NREC 2 -#define REC_NAME "time" -#define LVL_NAME "level" -#define NLVL 2 - -/* Names of things. */ -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" -#define UNITS "units" -#define DEGREES_EAST "degrees_east" -#define DEGREES_NORTH "degrees_north" - -/* These are used to calculate the values we expect to find. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* For the units attributes. */ -#define UNITS "units" -#define PRES_UNITS "hPa" -#define TEMP_UNITS "celsius" -#define LAT_UNITS "degrees_north" -#define LON_UNITS "degrees_east" -#define MAX_ATT_LEN 80 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); return 2;} - -int -main() -{ - int ncid, pres_varid, temp_varid; - int lat_varid, lon_varid; - - /* The start and count arrays will tell the netCDF library where to - read our data. */ - size_t start[NDIMS], count[NDIMS]; - - /* Program variables to hold the data we will read. We will only - need enough space to hold one timestep of data; one record. */ - float pres_in[NLVL][NLAT][NLON]; - float temp_in[NLVL][NLAT][NLON]; - - /* These program variables hold the latitudes and longitudes. */ - float lats[NLAT], lons[NLON]; - - /* Loop indexes. */ - int lvl, lat, lon, rec, i = 0; - - /* Error handling. */ - int retval; - - /* Open the file. */ - if ((retval = nc_open(FILE_NAME, NC_NOWRITE, &ncid))) - ERR(retval); - - /* Get the varids of the latitude and longitude coordinate - * variables. */ - if ((retval = nc_inq_varid(ncid, LAT_NAME, &lat_varid))) - ERR(retval); - if ((retval = nc_inq_varid(ncid, LON_NAME, &lon_varid))) - ERR(retval); - - /* Read the coordinate variable data. */ - if ((retval = nc_get_var_float(ncid, lat_varid, &lats[0]))) - ERR(retval); - if ((retval = nc_get_var_float(ncid, lon_varid, &lons[0]))) - ERR(retval); - - /* Check the coordinate variable data. */ - for (lat = 0; lat < NLAT; lat++) - if (lats[lat] != START_LAT + 5.*lat) - return 2; - for (lon = 0; lon < NLON; lon++) - if (lons[lon] != START_LON + 5.*lon) - return 2; - - /* Get the varids of the pressure and temperature netCDF - * variables. */ - if ((retval = nc_inq_varid(ncid, PRES_NAME, &pres_varid))) - ERR(retval); - if ((retval = nc_inq_varid(ncid, TEMP_NAME, &temp_varid))) - ERR(retval); - - /* Read the data. Since we know the contents of the file we know - * that the data arrays in this program are the correct size to - * hold one timestep. */ - count[0] = 1; - count[1] = NLVL; - count[2] = NLAT; - count[3] = NLON; - start[1] = 0; - start[2] = 0; - start[3] = 0; - - /* Read and check one record at a time. */ - for (rec = 0; rec < NREC; rec++) - { - start[0] = rec; - if ((retval = nc_get_vara_float(ncid, pres_varid, start, - count, &pres_in[0][0][0]))) - ERR(retval); - if ((retval = nc_get_vara_float(ncid, temp_varid, start, - count, &temp_in[0][0][0]))) - ERR(retval); - - /* Check the data. */ - i = 0; - for (lvl = 0; lvl < NLVL; lvl++) - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) - { - if (pres_in[lvl][lat][lon] != SAMPLE_PRESSURE + i) - return 2; - if (temp_in[lvl][lat][lon] != SAMPLE_TEMP + i) - return 2; - i++; - } - - } /* next record */ - - /* Close the file. */ - if ((retval = nc_close(ncid))) - ERR(retval); - - printf("*** SUCCESS reading example file pres_temp_4D.nc!\n"); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/pres_temp_4D_wr.c b/packages/trios/services/netcdf/tests/pres_temp_4D_wr.c deleted file mode 100644 index 697e8321514b..000000000000 --- a/packages/trios/services/netcdf/tests/pres_temp_4D_wr.c +++ /dev/null @@ -1,211 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This is an example program which writes some 4D pressure and - temperatures. It is intended to illustrate the use of the netCDF - C API. The companion program pres_temp_4D_rd.c shows how - to read the netCDF data file created by this program. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: pres_temp_4D_wr.c,v 1.4 2006/06/29 20:45:58 russ Exp $ -*/ - -#include -#include -#include - -/* This is the name of the data file we will create. */ -#define FILE_NAME "pres_temp_4D.nc" - -/* We are writing 4D data, a 2 x 6 x 12 lvl-lat-lon grid, with 2 - timesteps of data. */ -#define NDIMS 4 -#define NLAT 6 -#define NLON 12 -#define LAT_NAME "latitude" -#define LON_NAME "longitude" -#define NREC 2 -#define REC_NAME "time" -#define LVL_NAME "level" -#define NLVL 2 - -/* Names of things. */ -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" -#define UNITS "units" -#define DEGREES_EAST "degrees_east" -#define DEGREES_NORTH "degrees_north" - -/* These are used to construct some example data. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* For the units attributes. */ -#define UNITS "units" -#define PRES_UNITS "hPa" -#define TEMP_UNITS "celsius" -#define LAT_UNITS "degrees_north" -#define LON_UNITS "degrees_east" -#define MAX_ATT_LEN 80 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); return 2;} - -int -main() -{ - /* IDs for the netCDF file, dimensions, and variables. */ - int ncid, lon_dimid, lat_dimid, lvl_dimid, rec_dimid; - int lat_varid, lon_varid, pres_varid, temp_varid; - int dimids[NDIMS]; - - /* The start and count arrays will tell the netCDF library where to - write our data. */ - size_t start[NDIMS], count[NDIMS]; - - /* Program variables to hold the data we will write out. We will only - need enough space to hold one timestep of data; one record. */ - float pres_out[NLVL][NLAT][NLON]; - float temp_out[NLVL][NLAT][NLON]; - - /* These program variables hold the latitudes and longitudes. */ - float lats[NLAT], lons[NLON]; - - /* Loop indexes. */ - int lvl, lat, lon, rec, i = 0; - - /* Error handling. */ - int retval; - - /* Create some pretend data. If this wasn't an example program, we - * would have some real data to write, for example, model - * output. */ - for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5.*lat; - for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5.*lon; - - for (lvl = 0; lvl < NLVL; lvl++) - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) - { - pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + i; - temp_out[lvl][lat][lon] = SAMPLE_TEMP + i++; - } - - /* Create the file. */ - if ((retval = nc_create(FILE_NAME, NC_CLOBBER, &ncid))) - ERR(retval); - - /* Define the dimensions. The record dimension is defined to have - * unlimited length - it can grow as needed. In this example it is - * the time dimension.*/ - if ((retval = nc_def_dim(ncid, LVL_NAME, NLVL, &lvl_dimid))) - ERR(retval); - if ((retval = nc_def_dim(ncid, LAT_NAME, NLAT, &lat_dimid))) - ERR(retval); - if ((retval = nc_def_dim(ncid, LON_NAME, NLON, &lon_dimid))) - ERR(retval); - if ((retval = nc_def_dim(ncid, REC_NAME, NC_UNLIMITED, &rec_dimid))) - ERR(retval); - - /* Define the coordinate variables. We will only define coordinate - variables for lat and lon. Ordinarily we would need to provide - an array of dimension IDs for each variable's dimensions, but - since coordinate variables only have one dimension, we can - simply provide the address of that dimension ID (&lat_dimid) and - similarly for (&lon_dimid). */ - if ((retval = nc_def_var(ncid, LAT_NAME, NC_FLOAT, 1, &lat_dimid, - &lat_varid))) - ERR(retval); - if ((retval = nc_def_var(ncid, LON_NAME, NC_FLOAT, 1, &lon_dimid, - &lon_varid))) - ERR(retval); - - /* Assign units attributes to coordinate variables. */ - if ((retval = nc_put_att_text(ncid, lat_varid, UNITS, - strlen(DEGREES_NORTH), DEGREES_NORTH))) - ERR(retval); - if ((retval = nc_put_att_text(ncid, lon_varid, UNITS, - strlen(DEGREES_EAST), DEGREES_EAST))) - ERR(retval); - - /* The dimids array is used to pass the dimids of the dimensions of - the netCDF variables. Both of the netCDF variables we are - creating share the same four dimensions. In C, the - unlimited dimension must come first on the list of dimids. */ - dimids[0] = rec_dimid; - dimids[1] = lvl_dimid; - dimids[2] = lat_dimid; - dimids[3] = lon_dimid; - - /* Define the netCDF variables for the pressure and temperature - * data. */ - if ((retval = nc_def_var(ncid, PRES_NAME, NC_FLOAT, NDIMS, - dimids, &pres_varid))) - ERR(retval); - if ((retval = nc_def_var(ncid, TEMP_NAME, NC_FLOAT, NDIMS, - dimids, &temp_varid))) - ERR(retval); - - /* Assign units attributes to the netCDF variables. */ - if ((retval = nc_put_att_text(ncid, pres_varid, UNITS, - strlen(PRES_UNITS), PRES_UNITS))) - ERR(retval); - if ((retval = nc_put_att_text(ncid, temp_varid, UNITS, - strlen(TEMP_UNITS), TEMP_UNITS))) - ERR(retval); - - /* End define mode. */ - if ((retval = nc_enddef(ncid))) - ERR(retval); - - /* Write the coordinate variable data. This will put the latitudes - and longitudes of our data grid into the netCDF file. */ - if ((retval = nc_put_var_float(ncid, lat_varid, &lats[0]))) - ERR(retval); - if ((retval = nc_put_var_float(ncid, lon_varid, &lons[0]))) - ERR(retval); - - /* These settings tell netcdf to write one timestep of data. (The - setting of start[0] inside the loop below tells netCDF which - timestep to write.) */ - count[0] = 1; - count[1] = NLVL; - count[2] = NLAT; - count[3] = NLON; - start[1] = 0; - start[2] = 0; - start[3] = 0; - - /* Write the pretend data. This will write our surface pressure and - surface temperature data. The arrays only hold one timestep worth - of data. We will just rewrite the same data for each timestep. In - a real application, the data would change between timesteps. */ - for (rec = 0; rec < NREC; rec++) - { - start[0] = rec; - if ((retval = nc_put_vara_float(ncid, pres_varid, start, count, - &pres_out[0][0][0]))) - ERR(retval); - if ((retval = nc_put_vara_float(ncid, temp_varid, start, count, - &temp_out[0][0][0]))) - ERR(retval); - } - - /* Close the file. */ - if ((retval = nc_close(ncid))) - ERR(retval); - - printf("*** SUCCESS writing example file %s!\n", FILE_NAME); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/server_selection_test.cpp b/packages/trios/services/netcdf/tests/server_selection_test.cpp deleted file mode 100644 index e586125d19bc..000000000000 --- a/packages/trios/services/netcdf/tests/server_selection_test.cpp +++ /dev/null @@ -1,611 +0,0 @@ -#include -#include -#include - -#include -#include -#include -#include - -using namespace std; - -#define NDIMS 4 - -#define COUNT_DIM1 1 -#define COUNT_DIM2 8 -#define COUNT_DIM3 16 -#define COUNT_DIM4 16 - -#define DATATYPE_SIZE 4 - -#define PROC_DIM1 1 -#define PROC_DIM2 4 -#define PROC_DIM3 4 -#define PROC_DIM4 4 - -#define NUM_PROCS (PROC_DIM1*PROC_DIM2*PROC_DIM3*PROC_DIM4) -#define CLIENTS_PER_SERVER 4 -#define BYTES_PER_SERVER (COUNT_DIM1*COUNT_DIM2*COUNT_DIM3*COUNT_DIM4*DATATYPE_SIZE*CLIENTS_PER_SERVER) - -#define DIM1 (COUNT_DIM1*PROC_DIM1) -#define DIM2 (COUNT_DIM2*PROC_DIM2) -#define DIM3 (COUNT_DIM3*PROC_DIM3) -#define DIM4 (COUNT_DIM4*PROC_DIM4) - -struct start_offset_length_service { - long start[NDIMS]; /* coordinates of a contiguous subchunk */ - long count[NDIMS]; /* dimensions of a contiguous subchunk */ - long ndims; - long offset; - long length; - long datatype_size; - long service; -}; -typedef struct start_offset_length_service start_offset_length_service_t; - -struct consolidated_subchunks { - long start[NDIMS]; /* coordinates of a consolidated subchunk */ - long count[NDIMS]; /* dimensions of a consolidated subchunk */ - long ndims; - long offset_into_superchunk; /* offset into the buffer that is the chunk that contains this subchunk */ -}; -typedef struct consolidated_subchunks consolidated_subchunks_t; - -typedef list consolidated_subchunks_list_t; -typedef vector consolidated_subchunks_vector_t; -typedef vector::iterator consolidated_subchunks_vector_iterator_t; - -typedef map consolidated_subchunks_map_t; -typedef map::iterator consolidated_subchunks_map_iterator_t; - - -void calc_offset_length(long ndims, - long *dimlens, - long highest_contig_dim, - long current_dim, - long file_offset, - long *current_index, - start_offset_length_service_t *sols, - long *ol_index, - long *start, - long *count, - long datatype_size) -{ - if (current_dim < highest_contig_dim) { - current_index[current_dim]=start[current_dim]; - for (int i=0;i last) { - printf("ERROR - first > last"); - return(NULL); - } - if (first == last) { - consolidated_subchunks_t *c=new consolidated_subchunks_t; - memcpy(c->start, sols[first].start, sols[first].ndims*sizeof(long)); - memcpy(c->count, sols[first].count, sols[first].ndims*sizeof(long)); - c->ndims=sols[first].ndims; - long dim_product=1; - for (int i=0;ioffset_into_superchunk=(dim_product*sols[first].datatype_size); - - return(c); - } - for (long i=highest_contig_dim;istart, sols[first].start, sols[first].ndims*sizeof(long)); - memcpy(c->count, superchunk_count, sols[first].ndims*sizeof(long)); - for (long i=0;icount[i] = (sols[last].start[i]-sols[first].start[i])+1; - if (incomplete_dimensions[i] != -1) { - c->count[i] = incomplete_dimensions[i]; - } - } - c->ndims=sols[first].ndims; - long offset=0; - for (int i=0;ioffset_into_superchunk=(offset*sols[first].datatype_size); - } else { - printf("there are multiple(%ld) incomplete dimensions -- first subchunk is sols[%ld].start[", - num_incomplete_dimensions ,first); - for (int k=0;kinsert(v->end(), c); - - return(map); - } - - - - // start with a sanity check - long lower_dim_product=1; - for (int i=highest_contig_dim;iinsert(v->end(), c); - - first_adjacent_chunk=i+1; - } - - return(map); -} - -int main(int argc, char **argv) -{ - long ndims=NDIMS; - long dimlens[NDIMS]={DIM1, DIM2, DIM3, DIM4}; - long start[NDIMS]; - long count[NDIMS]={COUNT_DIM1,COUNT_DIM2,COUNT_DIM3,COUNT_DIM4}; - long num_servers; - - num_servers = (NUM_PROCS/CLIENTS_PER_SERVER); - if (NUM_PROCS%CLIENTS_PER_SERVER > 0) num_servers++; - - printf("%lu servers are required to service %d clients\n", num_servers, NUM_PROCS); - - for (long a=0;a=0;i--) { - if (count[i] < dimlens[i]) { - // the previous dim was the last contig dim - highest_contig_dimension = i; - break; - } - } - long num_contig_subchunks=1; - for (int i=0;ibegin(); - for (;map_iter != map->end(); map_iter++) { - consolidated_subchunks_vector_iterator_t vector_iter = (*map_iter).second->begin(); - for (;vector_iter != (*map_iter).second->end(); vector_iter++) { - printf("service(%02ld) super_start[%04ld,%04ld,%04ld,%04ld] start[%04ld,%04ld,%04ld,%04ld] count[%04ld,%04ld,%04ld,%04ld] offset_into_superchunk(%ld)\n", - (*map_iter).first, - start[0], start[1], start[2], start[3], - (*vector_iter)->start[0], (*vector_iter)->start[1], (*vector_iter)->start[2], (*vector_iter)->start[3], - (*vector_iter)->count[0], (*vector_iter)->count[1], (*vector_iter)->count[2], (*vector_iter)->count[3], - (*vector_iter)->offset_into_superchunk); - - } - - } - - - - -/* ************************************************************************* */ - - - -/* ************************************************************************* */ -// /* -// * (z+1)*y/CLIENT_PER_SERVER -// */ -// unsigned long dim_product=1; -// unsigned long count_product=1; -// unsigned long servers_required=0; -// if ((ndims == 4) && (count[0]) == 1) { -// // assume dim1 is the timestep dimension -// dim_product=dimlens[1]*dimlens[2]; -// count_product=count[1]*count[2]; -// servers_required=(dim_product/count_product)/CLIENTS_PER_SERVER; -// if (((dim_product/count_product)%CLIENTS_PER_SERVER) > 0) { -// servers_required++; -// } -// if (servers_required > num_servers) { -// printf("(dim_product(%lu)/count_product(%lu)/CLIENTS_PER_SERVER(%ul))(%lu) > num_servers(%lu) " -// "you need more servers.\n", -// dim_product, count_product, (dim_product/count_product/CLIENTS_PER_SERVER), num_servers); -// } -// svc_index = -// } -/* ************************************************************************* */ - -/* ************************************************************************* */ - /* this method of server selection is for a very specific array/client layout. - * assumptions: - * - ndims is 3 or 4. - * - if ndims == 4, then - * - dim1 is the timestep dimension. each client only works on one timestep - * at a time. therefore, dim1 may be >= 0, but count[0] is always 1. - * - dim2 is the Z dimension. we can split here. - * - dim3 is the Y dimension. we can split here. - * - dim4 is the X dimension. we cannot split processors on this dimension - * and get any decent aggregation. - * - if ndims == 3, then - * - if count[0] == 1, then - * - dim1 is the timestep dimension. each client only works on one timestep - * at a time. therefore, dim1 may be >= 0, but count[0] is always 1. - * - dim2 is the Y dimension. we can split here. - * - dim3 is the X dimension. we cannot split processors on this dimension - * and get any decent aggregation. - * - if count[0] > 1, then - * - assume dim1 is the Z dimension. we can split here. - * - dim2 is the Y dimension. we can split here. - * - dim3 is the X dimension. we cannot split processors on this dimension - * and get any decent aggregation. - */ - -// unsigned long dim_product=1; -// unsigned long count_product=1; -// long split_dim=-1; -// if ((ndims == 4) && (count[0]) == 1) { -// dim_product=dimlens[1]; -// count_product=count[1]; -// if ((dim_product/count_product) >= num_servers) { -// split_dim=1; -// } else { -// dim_product *= dimlens[2]; -// count_product *= count[2]; -// if ((dim_product/count_product) >= num_servers) { -// split_dim=2; -// } -// } -// } -/* ************************************************************************* */ - - -/* ************************************************************************* */ -// unsigned long dim_product=1; -// unsigned long count_product=1; -// long split_dim=-1; -// for (unsigned long i=0;i= num_servers) { -// // we have enough dimensions to divide up the data -// split_dim=i; -// break; -// } -// } -// if (split_dim == -1) { -// if ((a%COUNT_DIM == 0) && -// (b%COUNT_DIM == 0) && -// (c%COUNT_DIM == 0) && -// (d%COUNT_DIM == 0)) { -// printf("(dim_product(%lu)/count_product(%lu))(%lu) < num_servers(%lu) " -// "using default_server is your best bet.\n", -// dim_product, count_product, (dim_product/count_product), num_servers); -// } -// } -// unsigned long lower_dim_product=1; -// unsigned long start_offset=0; -// for (unsigned long i=0;i 0) { -// if (dim_product%num_servers > 0) { -// elements_per_server++; -// } -// break; -// } -// } -// if (elements_per_server == 0) { -// printf("elements_per_server==0. use default_svc.\n"); -// } else { -// unsigned long my_file_offset=0; -// unsigned long lower_dim_product=1; -// for (unsigned long i=0;i -#include -#include - -/* This is the name of the data file we will read. */ -#define FILE_NAME "sfc_pres_temp.nc" - -/* We are reading 2D data, a 6 x 12 lat-lon grid. */ -#define NDIMS 2 -#define NLAT 6 -#define NLON 12 - -#define LAT_NAME "latitude" -#define LON_NAME "longitude" -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" - -/* These are used to calculate the values we expect to find. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* For the units attributes. */ -#define UNITS "units" -#define PRES_UNITS "hPa" -#define TEMP_UNITS "celsius" -#define LAT_UNITS "degrees_north" -#define LON_UNITS "degrees_east" -#define MAX_ATT_LEN 80 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); return 2;} - -int -main() -{ - int ncid, pres_varid, temp_varid; - int lat_varid, lon_varid; - - /* We will read surface temperature and pressure fields. */ - float pres_in[NLAT][NLON]; - float temp_in[NLAT][NLON]; - - /* For the lat lon coordinate variables. */ - float lats_in[NLAT], lons_in[NLON]; - - /* To check the units attributes. */ - char pres_units_in[MAX_ATT_LEN], temp_units_in[MAX_ATT_LEN]; - char lat_units_in[MAX_ATT_LEN], lon_units_in[MAX_ATT_LEN]; - - /* We will learn about the data file and store results in these - program variables. */ - int ndims_in, nvars_in, ngatts_in, unlimdimid_in; - - /* Loop indexes. */ - int lat, lon; - - /* Error handling. */ - int retval; - - /* Open the file. */ - if ((retval = nc_open(FILE_NAME, NC_NOWRITE, &ncid))) - ERR(retval); - - /* There are a number of inquiry functions in netCDF which can be - used to learn about an unknown netCDF file. NC_INQ tells how - many netCDF variables, dimensions, and global attributes are in - the file; also the dimension id of the unlimited dimension, if - there is one. */ - if ((retval = nc_inq(ncid, &ndims_in, &nvars_in, &ngatts_in, - &unlimdimid_in))) - ERR(retval); - - /* In this case we know that there are 2 netCDF dimensions, 4 - netCDF variables, no global attributes, and no unlimited - dimension. */ - if (ndims_in != 2 || nvars_in != 4 || ngatts_in != 0 || - unlimdimid_in != -1) return 2; - - /* Get the varids of the latitude and longitude coordinate - * variables. */ - if ((retval = nc_inq_varid(ncid, LAT_NAME, &lat_varid))) - ERR(retval); - if ((retval = nc_inq_varid(ncid, LON_NAME, &lon_varid))) - ERR(retval); - - /* Read the coordinate variable data. */ - if ((retval = nc_get_var_float(ncid, lat_varid, &lats_in[0]))) - ERR(retval); - if ((retval = nc_get_var_float(ncid, lon_varid, &lons_in[0]))) - ERR(retval); - - /* Check the coordinate variable data. */ - for (lat = 0; lat < NLAT; lat++) - if (lats_in[lat] != START_LAT + 5.*lat) - return 2; - for (lon = 0; lon < NLON; lon++) - if (lons_in[lon] != START_LON + 5.*lon) - return 2; - - /* Get the varids of the pressure and temperature netCDF - * variables. */ - if ((retval = nc_inq_varid(ncid, PRES_NAME, &pres_varid))) - ERR(retval); - if ((retval = nc_inq_varid(ncid, TEMP_NAME, &temp_varid))) - ERR(retval); - - /* Read the data. Since we know the contents of the file we know - * that the data arrays in this program are the correct size to - * hold all the data. */ - if ((retval = nc_get_var_float(ncid, pres_varid, &pres_in[0][0]))) - ERR(retval); - if ((retval = nc_get_var_float(ncid, temp_varid, &temp_in[0][0]))) - ERR(retval); - - /* Check the data. */ - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) - if (pres_in[lat][lon] != SAMPLE_PRESSURE + (lon * NLAT + lat) || - temp_in[lat][lon] != SAMPLE_TEMP + .25 * (lon * NLAT + lat)) - return 2; - - /* Each of the netCDF variables has a "units" attribute. Let's read - them and check them. */ - if ((retval = nc_get_att_text(ncid, lat_varid, UNITS, lat_units_in))) - ERR(retval); - if (strncmp(lat_units_in, LAT_UNITS, strlen(LAT_UNITS))) - return 2; - - if ((retval = nc_get_att_text(ncid, lon_varid, UNITS, lon_units_in))) - ERR(retval); - if (strncmp(lon_units_in, LON_UNITS, strlen(LON_UNITS))) - return 2; - - if ((retval = nc_get_att_text(ncid, pres_varid, UNITS, pres_units_in))) - ERR(retval); - if (strncmp(pres_units_in, PRES_UNITS, strlen(PRES_UNITS))) - return 2; - - if ((retval = nc_get_att_text(ncid, temp_varid, UNITS, temp_units_in))) - ERR(retval); - if (strncmp(temp_units_in, TEMP_UNITS, strlen(TEMP_UNITS))) return 2; - - /* Close the file. */ - if ((retval = nc_close(ncid))) - ERR(retval); - - printf("*** SUCCESS reading example file sfc_pres_temp.nc!\n"); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/sfc_pres_temp_wr.c b/packages/trios/services/netcdf/tests/sfc_pres_temp_wr.c deleted file mode 100644 index 0b3941587fa2..000000000000 --- a/packages/trios/services/netcdf/tests/sfc_pres_temp_wr.c +++ /dev/null @@ -1,174 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This example writes some surface pressure and temperatures. It is - intended to illustrate the use of the netCDF C API. The companion - program sfc_pres_temp_rd.c shows how to read the netCDF data file - created by this program. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: sfc_pres_temp_wr.c,v 1.3 2006/06/13 20:46:16 ed Exp $ -*/ - -#include -#include -#include - -/* This is the name of the data file we will create. */ -#define FILE_NAME "sfc_pres_temp.nc" - -/* We are writing 2D data, a 6 x 12 lat-lon grid. We will need two - * netCDF dimensions. */ -#define NDIMS 2 -#define NLAT 6 -#define NLON 12 -#define LAT_NAME "latitude" -#define LON_NAME "longitude" - -/* Names of things. */ -#define PRES_NAME "pressure" -#define TEMP_NAME "temperature" -#define UNITS "units" -#define DEGREES_EAST "degrees_east" -#define DEGREES_NORTH "degrees_north" - -/* These are used to construct some example data. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); return 2;} - -int -main() -{ - int ncid, lon_dimid, lat_dimid, pres_varid, temp_varid; - -/* In addition to the latitude and longitude dimensions, we will also - create latitude and longitude netCDF variables which will hold the - actual latitudes and longitudes. Since they hold data about the - coordinate system, the netCDF term for these is: "coordinate - variables." */ - int lat_varid, lon_varid; - - int dimids[NDIMS]; - - /* We will write surface temperature and pressure fields. */ - float pres_out[NLAT][NLON]; - float temp_out[NLAT][NLON]; - float lats[NLAT], lons[NLON]; - - /* It's good practice for each netCDF variable to carry a "units" - * attribute. */ - char pres_units[] = "hPa"; - char temp_units[] = "celsius"; - - /* Loop indexes. */ - int lat, lon; - - /* Error handling. */ - int retval; - - /* Create some pretend data. If this wasn't an example program, we - * would have some real data to write, for example, model - * output. */ - for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5.*lat; - for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5.*lon; - - for (lat = 0; lat < NLAT; lat++) - for (lon = 0; lon < NLON; lon++) - { - pres_out[lat][lon] = SAMPLE_PRESSURE + (lon * NLAT + lat); - temp_out[lat][lon] = SAMPLE_TEMP + .25 * (lon * NLAT + lat); - } - - /* Create the file. */ - if ((retval = nc_create(FILE_NAME, NC_CLOBBER, &ncid))) - ERR(retval); - - /* Define the dimensions. */ - if ((retval = nc_def_dim(ncid, LAT_NAME, NLAT, &lat_dimid))) - ERR(retval); - if ((retval = nc_def_dim(ncid, LON_NAME, NLON, &lon_dimid))) - ERR(retval); - - /* Define coordinate netCDF variables. They will hold the - coordinate information, that is, the latitudes and longitudes. A - varid is returned for each.*/ - if ((retval = nc_def_var(ncid, LAT_NAME, NC_FLOAT, 1, &lat_dimid, - &lat_varid))) - ERR(retval); - if ((retval = nc_def_var(ncid, LON_NAME, NC_FLOAT, 1, &lon_dimid, - &lon_varid))) - ERR(retval); - - /* Define units attributes for coordinate vars. This attaches a - text attribute to each of the coordinate variables, containing - the units. Note that we are not writing a trailing NULL, just - "units", because the reading program may be fortran which does - not use null-terminated strings. In general it is up to the - reading C program to ensure that it puts null-terminators on - strings where necessary.*/ - if ((retval = nc_put_att_text(ncid, lat_varid, UNITS, - strlen(DEGREES_NORTH), DEGREES_NORTH))) - ERR(retval); - if ((retval = nc_put_att_text(ncid, lon_varid, UNITS, - strlen(DEGREES_EAST), DEGREES_EAST))) - ERR(retval); - - /* Define the netCDF variables. The dimids array is used to pass - the dimids of the dimensions of the variables.*/ - dimids[0] = lat_dimid; - dimids[1] = lon_dimid; - if ((retval = nc_def_var(ncid, PRES_NAME, NC_FLOAT, NDIMS, - dimids, &pres_varid))) - ERR(retval); - if ((retval = nc_def_var(ncid, TEMP_NAME, NC_FLOAT, NDIMS, - dimids, &temp_varid))) - ERR(retval); - - /* Define units attributes for vars. */ - if ((retval = nc_put_att_text(ncid, pres_varid, UNITS, - strlen(pres_units), pres_units))) - ERR(retval); - if ((retval = nc_put_att_text(ncid, temp_varid, UNITS, - strlen(temp_units), temp_units))) - ERR(retval); - - /* End define mode. */ - if ((retval = nc_enddef(ncid))) - ERR(retval); - - /* Write the coordinate variable data. This will put the latitudes - and longitudes of our data grid into the netCDF file. */ - if ((retval = nc_put_var_float(ncid, lat_varid, &lats[0]))) - ERR(retval); - if ((retval = nc_put_var_float(ncid, lon_varid, &lons[0]))) - ERR(retval); - - /* Write the pretend data. This will write our surface pressure and - surface temperature data. The arrays of data are the same size - as the netCDF variables we have defined. */ - if ((retval = nc_put_var_float(ncid, pres_varid, &pres_out[0][0]))) - ERR(retval); - if ((retval = nc_put_var_float(ncid, temp_varid, &temp_out[0][0]))) - ERR(retval); - - /* Close the file. */ - if ((retval = nc_close(ncid))) - ERR(retval); - - printf("*** SUCCESS writing example file sfc_pres_temp.nc!\n"); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/simple_xy_rd.c b/packages/trios/services/netcdf/tests/simple_xy_rd.c deleted file mode 100644 index 03c9a34b26fe..000000000000 --- a/packages/trios/services/netcdf/tests/simple_xy_rd.c +++ /dev/null @@ -1,69 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This is a simple example which reads a small dummy array, which was - written by simple_xy_wr.c. This is intended to illustrate the use - of the netCDF C API. - - This program is part of the netCDF tutorial: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: simple_xy_rd.c,v 1.9 2006/08/17 23:00:55 russ Exp $ -*/ -#include -#include -#include - -/* This is the name of the data file we will read. */ -#define FILE_NAME "simple_xy.nc" - -/* We are reading 2D data, a 6 x 12 grid. */ -#define NX 6 -#define NY 12 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} - -int -main() -{ - /* This will be the netCDF ID for the file and data variable. */ - int ncid, varid; - - int data_in[NX][NY]; - - /* Loop indexes, and error handling. */ - int x, y, retval; - - /* Open the file. NC_NOWRITE tells netCDF we want read-only access - * to the file.*/ - if ((retval = nc_open(FILE_NAME, NC_NOWRITE, &ncid))) - ERR(retval); - - /* Get the varid of the data variable, based on its name. */ - if ((retval = nc_inq_varid(ncid, "data", &varid))) - ERR(retval); - - /* Read the data. */ - if ((retval = nc_get_var_int(ncid, varid, &data_in[0][0]))) - ERR(retval); - - /* Check the data. */ - for (x = 0; x < NX; x++) - for (y = 0; y < NY; y++) - if (data_in[x][y] != x * NY + y) - return ERRCODE; - - /* Close the file, freeing all resources. */ - if ((retval = nc_close(ncid))) - ERR(retval); - - printf("*** SUCCESS reading example file %s!\n", FILE_NAME); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/simple_xy_wr.c b/packages/trios/services/netcdf/tests/simple_xy_wr.c deleted file mode 100644 index b35b2df272f7..000000000000 --- a/packages/trios/services/netcdf/tests/simple_xy_wr.c +++ /dev/null @@ -1,102 +0,0 @@ -/* This is part of the netCDF package. - Copyright 2006 University Corporation for Atmospheric Research/Unidata. - See COPYRIGHT file for conditions of use. - - This is a very simple example which writes a 2D array of - sample data. To handle this in netCDF we create two shared - dimensions, "x" and "y", and a netCDF variable, called "data". - - This example demonstrates the netCDF C API. This is part of the - netCDF tutorial, which can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-tutorial - - Full documentation of the netCDF C API can be found at: - http://www.unidata.ucar.edu/software/netcdf/docs/netcdf-c - - $Id: simple_xy_wr.c,v 1.11 2006/09/27 13:44:36 ed Exp $ -*/ -#include -#include -#include - -/* This is the name of the data file we will create. */ -#define FILE_NAME "simple_xy.nc" - -/* We are writing 2D data, a 6 x 12 grid. */ -#define NDIMS 2 -#define NX 6 -#define NY 12 - -/* Handle errors by printing an error message and exiting with a - * non-zero status. */ -#define ERRCODE 2 -#define ERR(e) {printf("Error: %s\n", nc_strerror(e)); exit(ERRCODE);} - -int -main() -{ - /* When we create netCDF variables and dimensions, we get back an - * ID for each one. */ - int ncid, x_dimid, y_dimid, varid; - int dimids[NDIMS]; - - /* This is the data array we will write. It will just be filled - * with a progression of numbers for this example. */ - int data_out[NX][NY]; - - /* Loop indexes, and error handling. */ - int x, y, retval; - - /* Create some pretend data. If this wasn't an example program, we - * would have some real data to write, for example, model - * output. */ - for (x = 0; x < NX; x++) - for (y = 0; y < NY; y++) - data_out[x][y] = x * NY + y; - - /* Always check the return code of every netCDF function call. In - * this example program, any retval which is not equal to NC_NOERR - * (0) will cause the program to print an error message and exit - * with a non-zero return code. */ - - /* Create the file. The NC_CLOBBER parameter tells netCDF to - * overwrite this file, if it already exists.*/ - if ((retval = nc_create(FILE_NAME, NC_CLOBBER, &ncid))) - ERR(retval); - - /* Define the dimensions. NetCDF will hand back an ID for each. */ - if ((retval = nc_def_dim(ncid, "x", NX, &x_dimid))) - ERR(retval); - if ((retval = nc_def_dim(ncid, "y", NY, &y_dimid))) - ERR(retval); - - /* The dimids array is used to pass the IDs of the dimensions of - * the variable. */ - dimids[0] = x_dimid; - dimids[1] = y_dimid; - - /* Define the variable. The type of the variable in this case is - * NC_INT (4-byte integer). */ - if ((retval = nc_def_var(ncid, "data", NC_INT, NDIMS, - dimids, &varid))) - ERR(retval); - - /* End define mode. This tells netCDF we are done defining - * metadata. */ - if ((retval = nc_enddef(ncid))) - ERR(retval); - - /* Write the pretend data to the file. Although netCDF supports - * reading and writing subsets of data, in this case we write all - * the data in one operation. */ - if ((retval = nc_put_var_int(ncid, varid, &data_out[0][0]))) - ERR(retval); - - /* Close the file. This frees up any internal netCDF resources - * associated with the file, and flushes any buffers. */ - if ((retval = nc_close(ncid))) - ERR(retval); - - printf("*** SUCCESS writing example file simple_xy.nc!\n"); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/test_read_coll.c b/packages/trios/services/netcdf/tests/test_read_coll.c deleted file mode 100644 index e2194ad13dcb..000000000000 --- a/packages/trios/services/netcdf/tests/test_read_coll.c +++ /dev/null @@ -1,355 +0,0 @@ -/*********************************************************** - * - * This test program reads a netCDF file, and then write it - * out to another netCDF file, using the parallel netCDF - * library using MPI-IO. The two files should be identical. - * - * Input File: "../data/test_int.nc" generated from original netcdf-3. - * Output File: "testread.nc" - * - * The CDL notation for the test dataset is shown below: - * - * netcdf test { - * - * dimensions: - * - * x = 100, y = 100, z = 100, time = NC_UNLIMITED; - * - * - * variables: // variable types, names, shapes, attributes - * - * int square(x, y); - * squre: description = "2-D integer array"; - * - * int cube(x,y,z); - * - * int time(time); // coordinate & record variable - * - * int xytime(time, x, y); // record variable - * - * - * // global attributes - * - * :title = "example netCDF dataset"; - * - * - * data: // data written for variables - * square = 0, 1, 2, 3, ... , 9999; - * cube = 0, 1, 2, 3, ... , 999999; - * time = 0, 1, 2, 3, ... , 99; // 100 records - * xytime = 0, 1, 2, 3, ... , 9999; // 100 records - * } - * - * - * - * This test uses collective APIs to read/write variable data and - * only deals with integer variables. - * - * This test assume # of processors = 4 - * - **********************************************************/ - -#include -#include -#include -#include -#include "testutils.h" - -/* Prototype for functions used only in this file */ -static void handle_error(int status); - -static void handle_error(int status) { - printf("%s\n", ncmpi_strerror(status)); -} - -int main(int argc, char **argv) { - - int i, j; - int status; - int ncid1, ncid2; - int ndims, nvars, ngatts, unlimdimid; - char name[NC_MAX_NAME]; - nc_type type, vartypes[NC_MAX_VARS]; - MPI_Offset attlen; - MPI_Offset dimlen, shape[NC_MAX_VAR_DIMS], varsize, start[NC_MAX_VAR_DIMS]; - void *valuep; - int dimids[NC_MAX_DIMS], varids[NC_MAX_VARS]; - int vardims[NC_MAX_VARS][NC_MAX_VAR_DIMS/16]; /* divided by 16 due to my memory limitation */ - int varndims[NC_MAX_VARS], varnatts[NC_MAX_VARS]; - params opts; - - int rank; - int nprocs; - MPI_Comm comm = MPI_COMM_WORLD; - - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (rank == 0) - fprintf(stderr, "Testing read ... "); - parse_read_args(argc, argv, rank, &opts); - - /********** START OF NETCDF ACCESS **************/ - - - /* Read a netCDF file and write it out to another file */ - - /** - * Open the input dataset - ncid1: - * File name: "../data/test_int.nc" - * Dataset API: Collective - * And create the output dataset - ncid2: - * File name: "testread.nc" - * Dataset API: Collective - */ - - status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid2); - if (status != NC_NOERR) handle_error(status); - - - /** - * Inquire the dataset definitions of input dataset AND - * Add dataset definitions for output dataset. - */ - - status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid); - if (status != NC_NOERR) handle_error(status); - - - /* Inquire global attributes, assume CHAR attributes. */ - - for (i = 0; i < ngatts; i++) { - status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen); - if (status != NC_NOERR) handle_error(status); - switch (type) { - case NC_CHAR: - valuep = (void *)malloc(attlen * sizeof(char)); - status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_SHORT: - valuep = (void *)malloc(attlen * sizeof(short)); - status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_INT: - valuep = (void *)malloc(attlen * sizeof(int)); - status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_FLOAT: - valuep = (void *)malloc(attlen * sizeof(float)); - status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_DOUBLE: - valuep = (void *)malloc(attlen * sizeof(double)); - status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - default: - ; - /* handle unexpected types */ - } - } - - /* Inquire dimension */ - - for (i = 0; i < ndims; i++) { - status = ncmpi_inq_dim(ncid1, i, name, &dimlen); - if (status != NC_NOERR) handle_error(status); - if (i == unlimdimid) - dimlen = NC_UNLIMITED; - status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i); - if (status != NC_NOERR) handle_error(status); - } - - /* Inquire variables */ - - for (i = 0; i < nvars; i++) { - status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i); - if (status != NC_NOERR) handle_error(status); - - /* var attributes, assume CHAR attributes */ - - for (j = 0; j < varnatts[i]; j++) { - status = ncmpi_inq_attname(ncid1, varids[i], j, name); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_inq_att (ncid1, varids[i], name, &type, &attlen); - if (status != NC_NOERR) handle_error(status); - switch (type) { - case NC_CHAR: - valuep = (void *)malloc(attlen * sizeof(char)); - status = ncmpi_get_att_text(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_SHORT: - valuep = (void *)malloc(attlen * sizeof(short)); - status = ncmpi_get_att_short(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_INT: - valuep = (void *)malloc(attlen * sizeof(int)); - status = ncmpi_get_att_int(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_FLOAT: - valuep = (void *)malloc(attlen * sizeof(float)); - status = ncmpi_get_att_float(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_DOUBLE: - valuep = (void *)malloc(attlen * sizeof(double)); - status = ncmpi_get_att_double(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - default: - ; - /* handle unexpected types */ - } - } - } - - /** - * End Define Mode (switch to data mode) for output dataset - * Dataset API: Collective - */ - - status = ncmpi_enddef(ncid2); - if (status != NC_NOERR) handle_error(status); - - /** - * Read data of variables from input dataset - * (ONLY DEAL WITH: NC_INT, NC_FLOAT, NC_DOUBLE for now) - * Write the data out to the corresponding variables in the output dataset - * - * Data Partition (Assume 4 processors): - * square: 2-D, (Block, *), 25*100 from 100*100 - * cube: 3-D, (Block, *, *), 25*100*100 from 100*100*100 - * xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100 - * time: 1-D, Block-wise, 25 from 100 - * - * Data Mode API: collective - */ - - for (i = 0; i < NC_MAX_VAR_DIMS; i++) - start[i] = 0; - for (i = 0; i < nvars; i++) { - varsize = 1; - for (j = 0; j < varndims[i]; j++) { - status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j); - if (status != NC_NOERR) handle_error(status); - if (j == 0) { - shape[j] /= nprocs; - start[j] = shape[j] * rank; - } - varsize *= shape[j]; - } - switch (vartypes[i]) { - case NC_CHAR: - break; - case NC_SHORT: - valuep = (void *)malloc(varsize * sizeof(short)); - status = ncmpi_get_vara_short_all(ncid1, i, start, shape, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_short_all(ncid2, varids[i], - start, shape, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_INT: - valuep = (void *)malloc(varsize * sizeof(int)); - status = ncmpi_get_vara_int_all(ncid1, i, start, shape, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int_all(ncid2, varids[i], - start, shape, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_FLOAT: - valuep = (void *)malloc(varsize * sizeof(float)); - status = ncmpi_get_vara_float_all(ncid1, i, start, shape, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_float_all(ncid2, varids[i], - start, shape, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_DOUBLE: - valuep = (void *)malloc(varsize * sizeof(double)); - status = ncmpi_get_vara_double_all(ncid1, i, start, shape, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_double_all(ncid2, varids[i], - start, shape, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - default: - ; - /* handle unexpected types */ - } - } - - status = ncmpi_sync(ncid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_sync(ncid2); - if (status != NC_NOERR) handle_error(status); - - /** - * Close the datasets - * Dataset API: collective - */ - - status = ncmpi_close(ncid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_close(ncid2); - if (status != NC_NOERR) handle_error(status); - - /******************* END OF NETCDF ACCESS ****************/ - -if (rank == 0) - fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname); - - MPI_Finalize(); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/test_read_indep.c b/packages/trios/services/netcdf/tests/test_read_indep.c deleted file mode 100644 index 3c9efdde2a9c..000000000000 --- a/packages/trios/services/netcdf/tests/test_read_indep.c +++ /dev/null @@ -1,364 +0,0 @@ -/*********************************************************** - * - * This test program reads a netCDF file, and then write it - * out to another netCDF file, using the parallel netCDF - * library using MPI-IO. The two files should be identical. - * - * Input File: "../data/test_int.nc" generated from original netcdf-3. - * Output File: "testread.nc" - * - * The CDL notation for the test dataset is shown below: - * - * netcdf test { - * - * dimensions: - * - * x = 100, y = 100, z = 100, time = NC_UNLIMITED; - * - * - * variables: // variable types, names, shapes, attributes - * - * int square(x, y); - * squre: description = "2-D integer array"; - * - * int cube(x,y,z); - * - * int time(time); // coordinate & record variable - * - * int xytime(time, x, y); // record variable - * - * - * // global attributes - * - * :title = "example netCDF dataset"; - * - * - * data: // data written for variables - * square = 0, 1, 2, 3, ... , 9999; - * cube = 0, 1, 2, 3, ... , 999999; - * time = 0, 1, 2, 3, ... , 99; // 100 records - * xytime = 0, 1, 2, 3, ... , 9999; // 100 records - * } - * - * - * - * This test uses non-collective APIs to read/write variable data and - * only deals with integer variables. - * - * This test assume # of processors = 4 - * - **********************************************************/ - -#include -#include -#include -#include -#include "testutils.h" - -/* Prototype for functions used only in this file */ -static void handle_error(int status); - -static void handle_error(int status) { - printf("%s\n", ncmpi_strerror(status)); -} - -int main(int argc, char **argv) { - - int i, j; - int status; - int ncid1, ncid2; - int ndims, nvars, ngatts, unlimdimid; - char name[NC_MAX_NAME]; - nc_type type, vartypes[NC_MAX_VARS]; - MPI_Offset attlen; - MPI_Offset dimlen, shape[NC_MAX_VAR_DIMS], varsize, start[NC_MAX_VAR_DIMS]; - void *valuep; - int dimids[NC_MAX_DIMS], varids[NC_MAX_VARS]; - int vardims[NC_MAX_VARS][NC_MAX_VAR_DIMS/16]; /* divided by 16 due to my memory limitation */ - int varndims[NC_MAX_VARS], varnatts[NC_MAX_VARS]; - params opts; - - int rank; - int nprocs; - MPI_Comm comm = MPI_COMM_WORLD; - - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (rank == 0) - fprintf(stderr, "Testing independent read ... "); - parse_read_args(argc, argv, rank, &opts); - - /********** START OF NETCDF ACCESS **************/ - - - /* Read a netCDF file and write it out to another file */ - - /** - * Open the input dataset - ncid1: - * File name: "../data/test_int.nc" - * Dataset API: Collective - * And create the output dataset - ncid2: - * File name: "testread.nc" - * Dataset API: Collective - */ - - status = ncmpi_open(comm, opts.infname, 0, MPI_INFO_NULL, &ncid1); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid2); - if (status != NC_NOERR) handle_error(status); - - - /** - * Inquire the dataset definitions of input dataset AND - * Add dataset definitions for output dataset. - */ - - status = ncmpi_inq(ncid1, &ndims, &nvars, &ngatts, &unlimdimid); - if (status != NC_NOERR) handle_error(status); - - - /* Inquire global attributes, assume CHAR attributes. */ - - for (i = 0; i < ngatts; i++) { - status = ncmpi_inq_attname(ncid1, NC_GLOBAL, i, name); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_inq_att (ncid1, NC_GLOBAL, name, &type, &attlen); - if (status != NC_NOERR) handle_error(status); - switch (type) { - case NC_CHAR: - valuep = (void *)malloc(attlen * sizeof(char)); - status = ncmpi_get_att_text(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_text (ncid2, NC_GLOBAL, name, attlen, (char *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_SHORT: - valuep = (void *)malloc(attlen * sizeof(short)); - status = ncmpi_get_att_short(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_short (ncid2, NC_GLOBAL, name, type, attlen, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_INT: - valuep = (void *)malloc(attlen * sizeof(int)); - status = ncmpi_get_att_int(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_int (ncid2, NC_GLOBAL, name, type, attlen, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_FLOAT: - valuep = (void *)malloc(attlen * sizeof(float)); - status = ncmpi_get_att_float(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_float (ncid2, NC_GLOBAL, name, type, attlen, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_DOUBLE: - valuep = (void *)malloc(attlen * sizeof(double)); - status = ncmpi_get_att_double(ncid1, NC_GLOBAL, name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_double (ncid2, NC_GLOBAL, name, type, attlen, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - default: - ; - /* handle unexpected types */ - } - } - - /* Inquire dimension */ - - for (i = 0; i < ndims; i++) { - status = ncmpi_inq_dim(ncid1, i, name, &dimlen); - if (status != NC_NOERR) handle_error(status); - if (i == unlimdimid) - dimlen = NC_UNLIMITED; - status = ncmpi_def_dim(ncid2, name, dimlen, dimids+i); - if (status != NC_NOERR) handle_error(status); - } - - /* Inquire variables */ - - for (i = 0; i < nvars; i++) { - status = ncmpi_inq_var (ncid1, i, name, vartypes+i, varndims+i, vardims[i], varnatts+i); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_def_var(ncid2, name, vartypes[i], varndims[i], vardims[i], varids+i); - if (status != NC_NOERR) handle_error(status); - - /* var attributes, assume CHAR attributes */ - - for (j = 0; j < varnatts[i]; j++) { - status = ncmpi_inq_attname(ncid1, varids[i], j, name); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_inq_att (ncid1, varids[i], name, &type, &attlen); - if (status != NC_NOERR) handle_error(status); - switch (type) { - case NC_CHAR: - valuep = (void *)malloc(attlen * sizeof(char)); - status = ncmpi_get_att_text(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_text (ncid2, varids[i], name, attlen, (char *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_SHORT: - valuep = (void *)malloc(attlen * sizeof(short)); - status = ncmpi_get_att_short(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_short (ncid2, varids[i], name, type, attlen, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_INT: - valuep = (void *)malloc(attlen * sizeof(int)); - status = ncmpi_get_att_int(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_int (ncid2, varids[i], name, type, attlen, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_FLOAT: - valuep = (void *)malloc(attlen * sizeof(float)); - status = ncmpi_get_att_float(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_float (ncid2, varids[i], name, type, attlen, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_DOUBLE: - valuep = (void *)malloc(attlen * sizeof(double)); - status = ncmpi_get_att_double(ncid1, varids[i], name, valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_double (ncid2, varids[i], name, type, attlen, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - default: - ; - /* handle unexpected types */ - } - } - } - - /** - * End Define Mode (switch to data mode) for output dataset - * Dataset API: Collective - */ - - status = ncmpi_enddef(ncid2); - if (status != NC_NOERR) handle_error(status); - - /** - * Read data of variables from input dataset (assume INT variables) - * Write the data out to the corresponding variables in the output dataset - * - * Data Partition (Assume 4 processors): - * square: 2-D, (Block, *), 25*100 from 100*100 - * cube: 3-D, (Block, *, *), 25*100*100 from 100*100*100 - * xytime: 3-D, (Block, *, *), 25*100*100 from 100*100*100 - * time: 1-D, Block-wise, 25 from 100 - * - * Data Mode API: non-collective - */ - - status = ncmpi_begin_indep_data(ncid1); - if (status != NC_NOERR) handle_error(status); - status =ncmpi_begin_indep_data(ncid2); - if (status != NC_NOERR) handle_error(status); - - for (i = 0; i < NC_MAX_VAR_DIMS; i++) - start[i] = 0; - for (i = 0; i < nvars; i++) { - varsize = 1; - for (j = 0; j < varndims[i]; j++) { - status = ncmpi_inq_dim(ncid1, vardims[i][j], name, shape + j); - if (status != NC_NOERR) handle_error(status); - if (j == 0) { - shape[j] /= nprocs; - start[j] = shape[j] * rank; - } - varsize *= shape[j]; - } - switch (vartypes[i]) { - case NC_CHAR: - break; - case NC_SHORT: - valuep = (void *)malloc(varsize * sizeof(short)); - status = ncmpi_get_vara_short(ncid1, i, start, shape, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_short(ncid2, varids[i], - start, shape, (short *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_INT: - valuep = (void *)malloc(varsize * sizeof(int)); - status = ncmpi_get_vara_int(ncid1, i, start, shape, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int(ncid2, varids[i], - start, shape, (int *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_FLOAT: - valuep = (void *)malloc(varsize * sizeof(float)); - status = ncmpi_get_vara_float(ncid1, i, start, shape, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_float(ncid2, varids[i], - start, shape, (float *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - case NC_DOUBLE: - valuep = (void *)malloc(varsize * sizeof(double)); - status = ncmpi_get_vara_double(ncid1, i, start, shape, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_double(ncid2, varids[i], - start, shape, (double *)valuep); - if (status != NC_NOERR) handle_error(status); - free(valuep); - break; - default: - ; - /* handle unexpected types */ - } - } - - status = ncmpi_end_indep_data(ncid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_end_indep_data(ncid2); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_sync(ncid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_sync(ncid2); - if (status != NC_NOERR) handle_error(status); - - /** - * Close the datasets - * Dataset API: collective - */ - - status = ncmpi_close(ncid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_close(ncid2); - if (status != NC_NOERR) handle_error(status); - - /******************* END OF NETCDF ACCESS ****************/ - -if (rank == 0) - fprintf(stderr, "OK\nInput file %s copied to: %s!\n", opts.infname, opts.outfname); - - MPI_Finalize(); - return 0; -} diff --git a/packages/trios/services/netcdf/tests/test_write_coll.c b/packages/trios/services/netcdf/tests/test_write_coll.c deleted file mode 100644 index e44ced9c6da2..000000000000 --- a/packages/trios/services/netcdf/tests/test_write_coll.c +++ /dev/null @@ -1,279 +0,0 @@ -/*********************************************************** - * - * This test program writes a netCDF file using the parallel - * netCDF library using MPI-IO. - * - * The output file is: "testwrite.nc" - * - * The CDL notation for the test dataset is shown below: - * - * netcdf test { - * - * dimensions: - * - * x = 100, y = 100, z = 100, time = NC_UNLIMITED; - * - * - * variables: // variable types, names, shapes, attributes - * - * int square(x, y); - * squre: description = "2-D integer array"; - * - * int cube(x,y,z); - * - * int time(time); // coordinate & record variable - * - * int xytime(time, x, y); // record variable - * - * - * // global attributes - * - * :title = "example netCDF dataset"; - * - * - * data: // data written for variables - * square = 0, 1, 2, 3, ... , 9999; - * cube = 0, 1, 2, 3, ... , 999999; - * time = 0, 1, 2, 3, ... , 99; // 100 records - * xytime = 0, 1, 2, 3, ... , 9999; // 100 records - * } - * - * - * - * This test uses collective APIs to write variable data and only - * deals with integer variables. - * - * This test assume # of processors = 4 - * - **********************************************************/ - - - -#include -#include -#include -#include -#include "testutils.h" - -/* Prototype for functions used only in this file */ -static void handle_error(int status); - -static void handle_error(int status) { - printf("%s\n", ncmpi_strerror(status)); -} - -int main(int argc, char **argv) { - - int i, j, k; - int status; - int ncid; - int dimid1, dimid2, dimid3, udimid; - int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1]; - MPI_Offset square_start[2], cube_start[3] = {0, 0, 0}; - MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50}; - MPI_Offset xytime_start[3] = {0, 0, 0}; - MPI_Offset xytime_count[3] = {100, 50, 50}; - MPI_Offset time_start[1], time_count[1] = {25}; - int square_id, cube_id, xytime_id, time_id; - static char title[] = "example netCDF dataset"; - static char description[] = "2-D integer array"; - int data[100][50][50], buffer[100]; - - int rank; - int nprocs; - MPI_Comm comm = MPI_COMM_WORLD; - double TotalTime; - double CreateTime; - double DataTime; - double SyncTime; - double CloseTime; - params opts; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (rank == 0) - fprintf(stderr, "Testing write ... "); - parse_write_args(argc, argv, rank, &opts); - - MPI_Barrier(MPI_COMM_WORLD); - TotalTime = MPI_Wtime(); - - /********** START OF NETCDF ACCESS **************/ - - /** - * Create the dataset - * File name: "testwrite.nc" - * Dataset API: Collective - */ - - MPI_Barrier(MPI_COMM_WORLD); - CreateTime = MPI_Wtime(); - status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid); - if (status != NC_NOERR) handle_error(status); - MPI_Barrier(MPI_COMM_WORLD); - CreateTime = MPI_Wtime() - CreateTime; - - - /** - * Create a global attribute: - * :title = "example netCDF dataset"; - */ - - MPI_Barrier(MPI_COMM_WORLD); - DataTime = MPI_Wtime(); - status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title", - strlen(title), title); - if (status != NC_NOERR) handle_error(status); - - /** - * Add 4 pre-defined dimensions: - * x = 100, y = 100, z = 100, time = NC_UNLIMITED - */ - - status = ncmpi_def_dim(ncid, "x", 100L, &dimid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_dim(ncid, "y", 100L, &dimid2); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_dim(ncid, "z", 100L, &dimid3); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid); - if (status != NC_NOERR) handle_error(status); - - /** - * Define the dimensionality and then add 4 variables: - * square(x, y), cube(x,y,z), time(time), xytime(time, x, y) - */ - - square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1; - square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2; - cube_dim[2] = dimid3; - xytime_dim[0] = udimid; - time_dim[0] = udimid; - status = ncmpi_def_var (ncid, "square", NC_INT, 2, square_dim, &square_id); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_var (ncid, "cube", NC_INT, 3, cube_dim, &cube_id); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_var (ncid, "xytime", NC_INT, 3, xytime_dim, &xytime_id); - if (status != NC_NOERR) handle_error(status); - - /** - * Add an attribute for variable: - * square: decsription = "2-D integer array" - */ - - status = ncmpi_put_att_text (ncid, square_id, "description", - strlen(description), description); - if (status != NC_NOERR) handle_error(status); - - /** - * End Define Mode (switch to data mode) - * Dataset API: Collective - */ - - status = ncmpi_enddef(ncid); - if (status != NC_NOERR) handle_error(status); - - /** - * Data Partition (Assume 4 processors): - * square: 2-D, (Block, Block), 50*50 from 100*100 - * cube: 3-D, (*, Block, Block), 100*50*50 from 100*100*100 - * xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100 - * time: 1-D, Block-wise, 25 from 100 - */ - - square_start[0] = cube_start[1] = xytime_start[1] = (rank/2) * 50; - square_start[1] = cube_start[2] = xytime_start[2] = (rank%2) * 50; - time_start[0] = (rank%4) * 25; - - - /** - * Packing data in the buffer - */ - - /* Data for variable: time */ - for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ ) - buffer[i - time_start[0]] = i; - - /* Data for variable: square, cube and xytime */ - for ( i = 0; i < 100; i++ ) - for ( j = square_start[0]; j < square_start[0]+square_count[0]; j++ ) - for ( k = square_start[1]; k < square_start[1]+square_count[1]; k++ ) - data[i][j-square_start[0]][k-square_start[1]] = i*100*100 + j*100 + k; - - /** - * Write data into variables: square, cube, time and xytime - * Access Method: subarray - * Data Mode API: collective - */ - - status = ncmpi_put_vara_int_all(ncid, square_id, - square_start, square_count, - &data[0][0][0]); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int_all(ncid, cube_id, - cube_start, cube_count, - &data[0][0][0]); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int_all(ncid, time_id, - time_start, time_count, - (void *)buffer); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int_all(ncid, xytime_id, - xytime_start, xytime_count, - &data[0][0][0]); - if (status != NC_NOERR) handle_error(status); - -/* -status = ncmpi_sync(ncid); -if (status != NC_NOERR) handle_error(status); -status = ncmpi_redef(ncid); -if (status != NC_NOERR) handle_error(status); -status = ncmpi_del_att(ncid, square_id, "description"); -if (status != NC_NOERR) handle_error(status); -status = ncmpi_enddef(ncid); -if (status != NC_NOERR) handle_error(status); -*/ - MPI_Barrier(MPI_COMM_WORLD); - DataTime = MPI_Wtime() - DataTime; - - MPI_Barrier(MPI_COMM_WORLD); - SyncTime = MPI_Wtime(); - status = ncmpi_sync(ncid); - if (status != NC_NOERR) handle_error(status); - MPI_Barrier(MPI_COMM_WORLD); - SyncTime = MPI_Wtime() - SyncTime; - - /** - * Close the dataset - * Dataset API: collective - */ - - MPI_Barrier(MPI_COMM_WORLD); - CloseTime = MPI_Wtime(); - status = ncmpi_close(ncid); - if (status != NC_NOERR) handle_error(status); - MPI_Barrier(MPI_COMM_WORLD); - CloseTime = MPI_Wtime() - CloseTime; - - /******************* END OF NETCDF ACCESS ****************/ - -MPI_Barrier(MPI_COMM_WORLD); -TotalTime = MPI_Wtime() - TotalTime; - -if (rank == 0) { - fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname); - fprintf(stderr, "Create Time = %10.8f\n", CreateTime); - fprintf(stderr, "Data Time = %10.8f\n", DataTime); - fprintf(stderr, "Sync Time = %10.8f\n", SyncTime); - fprintf(stderr, "Close Time = %10.8f\n", CloseTime); - fprintf(stderr, "Total Write Time = %10.8f\n", TotalTime); -} - - MPI_Finalize(); - return 0; -} - diff --git a/packages/trios/services/netcdf/tests/test_write_indep.c b/packages/trios/services/netcdf/tests/test_write_indep.c deleted file mode 100644 index 156586f4c3d8..000000000000 --- a/packages/trios/services/netcdf/tests/test_write_indep.c +++ /dev/null @@ -1,287 +0,0 @@ -/*********************************************************** - * - * This test program writes a netCDF file using the parallel - * netCDF library using MPI-IO. - * - * The output file is: "testwrite.nc" - * - * The CDL notation for the test dataset is shown below: - * - * netcdf test { - * - * dimensions: - * - * x = 100, y = 100, z = 100, time = NC_UNLIMITED; - * - * - * variables: // variable types, names, shapes, attributes - * - * int square(x, y); - * squre: description = "2-D integer array"; - * - * int cube(x,y,z); - * - * int time(time); // coordinate & record variable - * - * int xytime(time, x, y); // record variable - * - * - * // global attributes - * - * :title = "example netCDF dataset"; - * - * - * data: // data written for variables - * square = 0, 1, 2, 3, ... , 9999; - * cube = 0, 1, 2, 3, ... , 999999; - * time = 0, 1, 2, 3, ... , 99; // 100 records - * xytime = 0, 1, 2, 3, ... , 9999; // 100 records - * } - * - * - * - * This test uses non-collective APIs to write variable data and only - * deals with integer variables. - * - * This test assume # of processors = 4 - * - **********************************************************/ - - - -#include -#include -#include -#include -#include "testutils.h" - - -/* Prototype for functions used only in this file */ -static void handle_error(int status); - -static void handle_error(int status) { - printf("%s\n", ncmpi_strerror(status)); -} - -int main(int argc, char **argv) { - - int i, j, k; - int status; - int ncid; - int dimid1, dimid2, dimid3, udimid; - int square_dim[2], cube_dim[3], xytime_dim[3], time_dim[1]; - MPI_Offset square_start[2], cube_start[3] = {0, 0, 0}; - MPI_Offset square_count[2] = {50, 50}, cube_count[3] = {100, 50, 50}; - MPI_Offset xytime_start[3] = {0, 0, 0}; - MPI_Offset xytime_count[3] = {100, 50, 50}; - MPI_Offset time_start[1], time_count[1] = {25}; - int square_id, cube_id, xytime_id, time_id; - static char title[] = "example netCDF dataset"; -/* static char description[] = "2-D integer array"; */ - int data[100][50][50], buffer[100]; - - int rank; - int nprocs; - MPI_Comm comm = MPI_COMM_WORLD; - double TotalTime; - double CreateTime; - double DataTime; - double SyncTime; - double CloseTime; - params opts; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - if (rank == 0) - fprintf(stderr, "Testing independent write ... "); - parse_write_args(argc, argv, rank, &opts); - - MPI_Barrier(MPI_COMM_WORLD); - TotalTime = MPI_Wtime(); - - /********** START OF NETCDF ACCESS **************/ - - /** - * Create the dataset - * File name: "testwrite.nc" - * Dataset API: Collective - */ - - MPI_Barrier(MPI_COMM_WORLD); - CreateTime = MPI_Wtime(); - status = ncmpi_create(comm, opts.outfname, NC_CLOBBER, MPI_INFO_NULL, &ncid); - if (status != NC_NOERR) handle_error(status); - MPI_Barrier(MPI_COMM_WORLD); - CreateTime = MPI_Wtime() - CreateTime; - - - /** - * Create a global attribute: - * :title = "example netCDF dataset"; - */ - - MPI_Barrier(MPI_COMM_WORLD); - DataTime = MPI_Wtime(); - status = ncmpi_put_att_text (ncid, NC_GLOBAL, "title", - strlen(title), title); - if (status != NC_NOERR) handle_error(status); - - /** - * Add 4 pre-defined dimensions: - * x = 100, y = 100, z = 100, time = NC_UNLIMITED - */ - - status = ncmpi_def_dim(ncid, "x", 100L, &dimid1); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_dim(ncid, "y", 100L, &dimid2); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_dim(ncid, "z", 100L, &dimid3); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_dim(ncid, "time", NC_UNLIMITED, &udimid); - if (status != NC_NOERR) handle_error(status); - - /** - * Define the dimensionality and then add 4 variables: - * square(x, y), cube(x,y,z), time(time), xytime(time, x, y) - */ - - square_dim[0] = cube_dim[0] = xytime_dim[1] = dimid1; - square_dim[1] = cube_dim[1] = xytime_dim[2] = dimid2; - cube_dim[2] = dimid3; - xytime_dim[0] = udimid; - time_dim[0] = udimid; - status = ncmpi_def_var (ncid, "square", NC_INT, 2, square_dim, &square_id); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_var (ncid, "cube", NC_INT, 3, cube_dim, &cube_id); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_var (ncid, "time", NC_INT, 1, time_dim, &time_id); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_def_var (ncid, "xytime", NC_INT, 3, xytime_dim, &xytime_id); - if (status != NC_NOERR) handle_error(status); - - /** - * Add an attribute for variable: - * square: decsription = "2-D integer array" - */ - -/* - status = ncmpi_put_att_text (ncid, square_id, "description", - strlen(description), description); - if (status != NC_NOERR) handle_error(status); -*/ - - /** - * End Define Mode (switch to data mode) - * Dataset API: Collective - */ - - status = ncmpi_enddef(ncid); - if (status != NC_NOERR) handle_error(status); - - /** - * Data Partition (Assume 4 processors): - * square: 2-D, (Block, Block), 50*50 from 100*100 - * cube: 3-D, (*, Block, Block), 100*50*50 from 100*100*100 - * xytime: 3-D, (*, Block, Block), 100*50*50 from 100*100*100 - * time: 1-D, Block-wise, 25 from 100 - */ - - square_start[0] = cube_start[1] = xytime_start[1] = (rank/2) * 50; - square_start[1] = cube_start[2] = xytime_start[2] = (rank%2) * 50; - time_start[0] = (rank%4) * 25; - - - /** - * Packing data in the buffer - */ - - /* Data for variable: time */ - for ( i = time_start[0]; i < time_start[0] + time_count[0]; i++ ) - buffer[i - time_start[0]] = i; - - /* Data for variable: square, cube and xytime */ - for ( i = 0; i < 100; i++ ) - for ( j = square_start[0]; j < square_start[0]+square_count[0]; j++ ) - for ( k = square_start[1]; k < square_start[1]+square_count[1]; k++ ) - data[i][j-square_start[0]][k-square_start[1]] = i*100*100 + j*100 + k; - - /** - * Write data into variables: square, cube, time and xytime - * Access Method: subarray - * Data Mode API: non-collective - */ - - status = ncmpi_begin_indep_data(ncid); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_put_vara_int(ncid, square_id, - square_start, square_count, - &data[0][0][0]); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int(ncid, cube_id, - cube_start, cube_count, - &data[0][0][0]); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int(ncid, time_id, - time_start, time_count, - (void *)buffer); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_vara_int(ncid, xytime_id, - xytime_start, xytime_count, - &data[0][0][0]); - if (status != NC_NOERR) handle_error(status); - -/* - status = ncmpi_end_indep_data(ncid); - if (status != NC_NOERR) handle_error(status); - - status = ncmpi_redef(ncid); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_put_att_text (ncid, square_id, "description", - strlen(description), description); - if (status != NC_NOERR) handle_error(status); - status = ncmpi_enddef(ncid); - if (status != NC_NOERR) handle_error(status); -*/ - MPI_Barrier(MPI_COMM_WORLD); - DataTime = MPI_Wtime() - DataTime; - - MPI_Barrier(MPI_COMM_WORLD); - SyncTime = MPI_Wtime(); - status = ncmpi_sync(ncid); - if (status != NC_NOERR) handle_error(status); - MPI_Barrier(MPI_COMM_WORLD); - SyncTime = MPI_Wtime() - SyncTime; - - /** - * Close the dataset - * Dataset API: collective - */ - - MPI_Barrier(MPI_COMM_WORLD); - CloseTime = MPI_Wtime(); - status = ncmpi_close(ncid); - if (status != NC_NOERR) handle_error(status); - MPI_Barrier(MPI_COMM_WORLD); - CloseTime = MPI_Wtime() - CloseTime; - - /******************* END OF NETCDF ACCESS ****************/ - - MPI_Barrier(MPI_COMM_WORLD); - TotalTime = MPI_Wtime() - TotalTime; - - if (rank == 0) { - fprintf(stderr, "OK\nFile written to: %s!\n", opts.outfname); - fprintf(stderr, "Create Time = %10.8f\n", CreateTime); - fprintf(stderr, "Data Time = %10.8f\n", DataTime); - fprintf(stderr, "Sync Time = %10.8f\n", SyncTime); - fprintf(stderr, "Close Time = %10.8f\n", CloseTime); - fprintf(stderr, "Total Write Time = %10.8f\n", TotalTime); - } - - MPI_Finalize(); - return 0; -} - diff --git a/packages/trios/services/netcdf/tests/testutils.c b/packages/trios/services/netcdf/tests/testutils.c deleted file mode 100644 index f156ff693b55..000000000000 --- a/packages/trios/services/netcdf/tests/testutils.c +++ /dev/null @@ -1,48 +0,0 @@ -#include -#include -#include -#include "mpi.h" -#include "testutils.h" - -void parse_read_args(int argc, char **argv, int rank, params *p) -{ - int inlen, outlen; - if ( rank == 0 ) { - if (argc == 3 ) { - strncpy(p->infname, argv[1], PATH_MAX); - strncpy(p->outfname, argv[2], PATH_MAX); - } else if (argc == 0) { - strncpy(p->infname, "pvfs:../data/test_double.nc", - PATH_MAX); - strncpy(p->outfname, "pvfs:testread.nc", PATH_MAX); - } else { - fprintf(stderr, "Usage: %s: \n", - argv[0]); - MPI_Abort(MPI_COMM_WORLD, 1); - } - inlen = strlen(p->infname); - outlen = strlen(p->outfname); - } - MPI_Bcast(&inlen, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(p->infname, inlen+1, MPI_CHAR, 0, MPI_COMM_WORLD); - MPI_Bcast(&outlen, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(p->outfname, outlen+1, MPI_CHAR, 0, MPI_COMM_WORLD); -} - -void parse_write_args(int argc, char **argv, int rank, params *p) -{ - int outlen; - if ( rank == 0 ) { - if (argc == 2 ) { - strncpy(p->outfname, argv[1], PATH_MAX); - } else if (argc == 0) { - strncpy(p->outfname, "pvfs:testwrite.nc", PATH_MAX); - } else { - fprintf(stderr, "Usage: %s: \n", argv[0]); - MPI_Abort(MPI_COMM_WORLD, 1); - } - outlen = strlen(p->outfname); - } - MPI_Bcast(&outlen, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(p->outfname, outlen+1, MPI_CHAR, 0, MPI_COMM_WORLD); -} diff --git a/packages/trios/services/netcdf/tests/testutils.h b/packages/trios/services/netcdf/tests/testutils.h deleted file mode 100644 index 6fdc78fa8e65..000000000000 --- a/packages/trios/services/netcdf/tests/testutils.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _UTILS_H -#define _UTILS_H - -#include -#include - -typedef struct { - char infname[PATH_MAX]; - char outfname[PATH_MAX]; -} params; - -void parse_read_args(int argc, char **argv, int rank, params *p); -void parse_write_args(int argc, char **argv, int rank, params *p); -#endif diff --git a/packages/trios/tests/CMakeLists.txt b/packages/trios/tests/CMakeLists.txt deleted file mode 100644 index 80173e510105..000000000000 --- a/packages/trios/tests/CMakeLists.txt +++ /dev/null @@ -1,57 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - - -TRIBITS_SUBPACKAGE(tests) - -TRIBITS_ADD_TEST_DIRECTORIES(unit) -IF (TPL_ENABLE_MPI) - TRIBITS_ADD_TEST_DIRECTORIES(nssi_send) - TRIBITS_ADD_TEST_DIRECTORIES(opreg) - TRIBITS_ADD_TEST_DIRECTORIES(injection) - TRIBITS_ADD_TEST_DIRECTORIES(multicast) - TRIBITS_ADD_TEST_DIRECTORIES(netperf) - TRIBITS_ADD_TEST_DIRECTORIES(getservice) -ENDIF (TPL_ENABLE_MPI) - -TRIBITS_SUBPACKAGE_POSTPROCESS() diff --git a/packages/trios/tests/cmake/Dependencies.cmake b/packages/trios/tests/cmake/Dependencies.cmake deleted file mode 100644 index 7ab7a9b6f047..000000000000 --- a/packages/trios/tests/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS MPI Pthread) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/getservice/CMakeLists.txt b/packages/trios/tests/getservice/CMakeLists.txt deleted file mode 100644 index 1312f386abb0..000000000000 --- a/packages/trios/tests/getservice/CMakeLists.txt +++ /dev/null @@ -1,87 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -############# Special exception for rpcgen extra commas in enum ########## -IF (CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") -ENDIF (CMAKE_COMPILER_IS_GNUCXX) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -# We want the binary directory to be a system directory because it contains -# code generated by rpcgen, may have many warnings. -#INCLUDE_DIRECTORIES(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}) - - -TRIBITS_ADD_EXECUTABLE( - GetServiceTest - SOURCES getservice_server.cpp getservice_test.cpp getservice_client.cpp getservice_debug.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -SET(EXTRA_TEST_ARGS "") - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -SET(DEFAULT_ARGS "--result-file=getservice-result.out ") -SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file-mode=a ") -SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-trials=5 ") -SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-reqs=10 ") - -TRIBITS_ADD_TEST( - GetServiceTest - NOEXEPREFIX - NAME GetServiceTest ${testname} - ARGS "${DEFAULT_ARGS} ${EXTRA_TEST_ARGS}" - COMM serial mpi - NUM_MPI_PROCS 2) diff --git a/packages/trios/tests/getservice/cmake/Dependencies.cmake b/packages/trios/tests/getservice/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/tests/getservice/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/getservice/getservice_client.cpp b/packages/trios/tests/getservice/getservice_client.cpp deleted file mode 100644 index bbada195afe7..000000000000 --- a/packages/trios/tests/getservice/getservice_client.cpp +++ /dev/null @@ -1,542 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * getservice_client.cpp - * - * Created on: Jul 20, 2012 - * Author: thkorde - */ - -/** - * @defgroup getservice_client Data Transfer Client - * - * @ingroup getservice_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nssi_debug.h" - -#include "getservice_test.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - - -log_level client_debug_level = LOG_UNDEFINED; - -extern int print_args( - std::ostream &out, - const struct getservice_args &args, - const char *prefix); - - -int read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return(-1); - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return(1); - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(client_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); - - return(0); -} - - -int -compare_services(nssi_service &service1, nssi_service &service2) -{ - int rc=NSSI_OK; - - if (service1.rpc_encode != service2.rpc_encode) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - - if (strcmp(service1.svc_host.url, service2.svc_host.url)) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.svc_host.peer.transport_id != service2.svc_host.peer.transport_id) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - switch (service1.svc_host.peer.transport_id) { - case NNTI_TRANSPORT_PORTALS: - if (service1.svc_host.peer.NNTI_remote_process_t_u.portals.nid != service2.svc_host.peer.NNTI_remote_process_t_u.portals.nid) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.svc_host.peer.NNTI_remote_process_t_u.portals.pid != service2.svc_host.peer.NNTI_remote_process_t_u.portals.pid) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_IB: - if (service1.svc_host.peer.NNTI_remote_process_t_u.ib.addr != service2.svc_host.peer.NNTI_remote_process_t_u.ib.addr) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.svc_host.peer.NNTI_remote_process_t_u.ib.port != service2.svc_host.peer.NNTI_remote_process_t_u.ib.port) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.svc_host.peer.NNTI_remote_process_t_u.ib.qpn != service2.svc_host.peer.NNTI_remote_process_t_u.ib.qpn) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_GEMINI: - if (service1.svc_host.peer.NNTI_remote_process_t_u.gni.addr != service2.svc_host.peer.NNTI_remote_process_t_u.gni.addr) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.svc_host.peer.NNTI_remote_process_t_u.gni.port != service2.svc_host.peer.NNTI_remote_process_t_u.gni.port) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.svc_host.peer.NNTI_remote_process_t_u.gni.inst_id != service2.svc_host.peer.NNTI_remote_process_t_u.gni.inst_id) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_MPI: - if (service1.svc_host.peer.NNTI_remote_process_t_u.mpi.rank != service2.svc_host.peer.NNTI_remote_process_t_u.mpi.rank) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_LOCAL: - case NNTI_TRANSPORT_NULL: - break; - } - - if (service1.req_addr.transport_id != service2.req_addr.transport_id) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - switch (service1.req_addr.buffer_owner.peer.transport_id) { - case NNTI_TRANSPORT_PORTALS: - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.portals.nid != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.portals.nid) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.portals.pid != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.portals.pid) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_IB: - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.ib.addr != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.ib.addr) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.ib.port != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.ib.port) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.ib.qpn != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.ib.qpn) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_GEMINI: - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.gni.addr != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.gni.addr) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.gni.port != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.gni.port) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.gni.inst_id != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.gni.inst_id) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_MPI: - if (service1.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.mpi.rank != service2.req_addr.buffer_owner.peer.NNTI_remote_process_t_u.mpi.rank) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_LOCAL: - case NNTI_TRANSPORT_NULL: - break; - } - - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - switch (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].transport_id) { - case NNTI_TRANSPORT_PORTALS: - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.buffer_id) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.match_bits) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.size != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.portals.size) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_IB: - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.buf) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.key != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.key) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.size != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.size) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_buf != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_buf) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_size != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.ib.ack_size) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_GEMINI: - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.type != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.type) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.buf) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword1 != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword1) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword2 != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.mem_hdl.qword2) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.size) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.wc_addr != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.wc_addr) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.wc_mem_hdl.qword1 != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.wc_mem_hdl.qword1) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.wc_mem_hdl.qword2 != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.gni.wc_mem_hdl.qword2) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_MPI: - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.cmd_tag) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.get_data_tag != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.get_data_tag) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.put_data_tag != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.put_data_tag) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.size != service2.req_addr.buffer_segments.NNTI_remote_addr_array_t_val[0].NNTI_remote_addr_t_u.mpi.size) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - break; - case NNTI_TRANSPORT_LOCAL: - case NNTI_TRANSPORT_NULL: - break; - } - - if (service1.req_addr.ops != service2.req_addr.ops) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.payload_size != service2.req_addr.payload_size) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - if (service1.req_addr.payload != service2.req_addr.payload) { - log_error(getservice_debug_level, "service compare FAILED"); - rc=NSSI_EINVAL; - goto out; - } - -out: - return(rc); -} - - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param getservice_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -getservice_client_main (struct getservice_args &args, nssi_service &getservice_svc, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int i,j; - double time; - double start_time; - std::ofstream result_stream; - log_level debug_level = getservice_debug_level; - int client_rank, client_size; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - std::vector < nssi_request > reqs(num_reqs); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - log_debug(debug_level, "%d: Starting experiment loop", client_rank); - - /* loop over the experiments (count == num_trials, num_reqs == ops_per_trial) */ - for (i=0; i -#include - -#include -#include -#include -#include -#include -#include - - -log_level service_debug_level = LOG_UNDEFINED; - - - -/** - * @brief The NSSI getservice-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int getservice_server_main(struct getservice_args &args, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service getservice_svc; - int server_rank; - - MPI_Comm_rank(server_comm, &server_rank); - - /* options that can be overriden by the command-line */ - std::string server_url(NSSI_URL_LEN, '\0'); /* NNTI-style url of the server */ - std::string logfile(""); - - - memset(&getservice_svc, 0, sizeof(nssi_service)); - - - /* initialize the nssi service */ - rc = nssi_service_init((nssi_rpc_transport)args.transport, NSSI_SHORT_REQUEST_SIZE, &getservice_svc); - if (rc != NSSI_OK) { - log_error(getservice_debug_level, "could not init getservice_svc: %s", - nssi_err_str(rc)); - return -1; - } - - // Get the Server URL - std::string url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &url[0], NSSI_URL_LEN); - - - // Set the maxumum number of requests to handle (-1 == infinite) - getservice_svc.max_reqs = -1; - - log_debug(getservice_debug_level, "Starting Server: url = %s", url.c_str()); - - // Tell the NSSI server to output log data - //rpc_debug_level = getservice_debug_level; - - // start processing requests, the client will send a request to exit when done - rc = nssi_service_start(&getservice_svc); - if (rc != NSSI_OK) { - log_info(getservice_debug_level, "exited getservice_svc: %s", - nssi_err_str(rc)); - } - - sleep(5); - - /* shutdown the getservice_svc */ - log_debug(getservice_debug_level, "shutting down service library"); - nssi_service_fini(&getservice_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/getservice/getservice_test.cpp b/packages/trios/tests/getservice/getservice_test.cpp deleted file mode 100644 index 4312b744e840..000000000000 --- a/packages/trios/tests/getservice/getservice_test.cpp +++ /dev/null @@ -1,543 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * getservice_test.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include "getservice_test.h" -#include "getservice_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -int getservice_server_main(struct getservice_args &args, MPI_Comm server_comm); -int getservice_client_main (struct getservice_args &args, nssi_service &getservice_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct getservice_args &args, - const char *prefix) -{ - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url << std::endl; - - if (args.client_flag) { - out << prefix << " \ttransport = " << args.transport_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - nssi_service getservice_svc; - - int transport_index=-1; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct getservice_args args; - - - const int nssi_transport_list[] = { - NSSI_RPC_PTL, - NSSI_RPC_PTL, - NSSI_RPC_IB, - NSSI_RPC_IB, - NSSI_RPC_GEMINI, - NSSI_RPC_GEMINI, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGQPAMI, - NSSI_RPC_BGQPAMI, - NSSI_RPC_MPI}; - - const int num_nssi_transports = 11; - const int nssi_transport_vals[] = { - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10 - }; - const char * nssi_transport_names[] = { - "portals", - "ptl", - "infiniband", - "ib", - "gemini", - "gni", - "bgpdcmf", - "dcmf", - "bgqpami", - "pami", - "mpi" - }; - - - // Initialize arguments - args.transport=NSSI_DEFAULT_TRANSPORT; - args.delay = 1; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.timeout = 500; - args.num_retries = 5; - args.server_url = ""; - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("server", "no-server", &args.server_flag, "Run the server" ); - parser.setOption("client", "no-client", &args.client_flag, "Run the client"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("server-url", &args.server_url, "URL client uses to find the server"); - parser.setOption("server-url-file", &args.url_file, "File that has URL client uses to find server"); - - // Set an enumeration command line option for the NNTI transport - parser.setOption("transport", &transport_index, num_nssi_transports, nssi_transport_vals, nssi_transport_names, - "NSSI transports (not all are available on every platform): \n" - "\t\t\tportals|ptl : Cray or Schutt\n" - "\t\t\tinfiniband|ib : libibverbs\n" - "\t\t\tgemini|gni : Cray libugni (Gemini or Aries)\n" - "\t\t\tbgpdcmf|dcmf : IBM BG/P DCMF\n" - "\t\t\tbgqpami|pami : IBM BG/Q PAMI\n" - "\t\t\tmpi : isend/irecv implementation\n" - ); - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(LOG_ALL, "transport_index=%d", transport_index); - if (transport_index > -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting getservice-service test", rank); - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - if (args.client_flag && args.server_flag) { - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - - int color = (rank == 0) ? 0 : 1; // only one server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - // std::cout << "rank=" << rank << "/" << np << ", color=" << color << - // ", new_rank=" << newrank << "/" << newsize << std::endl << std::endl; - // - // std::cout << "my_url=" << my_url << ", server_url=" << args.server_url << std::endl; - } - else { - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - } - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - if (args.server_flag && !args.server_url.empty()) { - // use the server URL as suggested URL - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, args.server_url.c_str()); - } - else { - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - } - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // Broadcast the server URL to all the clients - args.server_url.resize(NSSI_URL_LEN, '\0'); - if (args.server_flag && args.client_flag) { - args.server_url = my_url; - MPI_Bcast(&args.server_url[0], args.server_url.size(), MPI_CHAR, 0, MPI_COMM_WORLD); - } - - else if (!args.server_flag && args.client_flag){ - if (args.server_url.empty()) { - - // check to see if we're supposed to get the URL from a file - if (!args.url_file.empty()) { - // Fetch the server URL from a file - sleep(1); - log_debug(debug_level, "Reading from file %s", args.url_file.c_str()); - std::ifstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - if (urlfile.good()) - getline(urlfile, args.server_url); - } - else { - log_error(debug_level, "Failed to open server_url_file=%s", args.url_file.c_str()); - exit(1); - } - urlfile.close(); - log_debug(debug_level, "URL = %s", args.server_url.c_str()); - } - else { - log_error(debug_level, "Need to set --server-url=[ADDR] or --server-url-file=[PATH]"); - } - } - } - - else if (args.server_flag && !args.client_flag) { - args.server_url = my_url; - - // If the url_file value is set, write the url to a file - if (!args.url_file.empty()) { - std::ofstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - urlfile << args.server_url.c_str() << std::endl; - } - urlfile.close(); - log_debug(debug_level, "Wrote url to file %s", args.url_file.c_str()); - } - } - - - - // Set the debug level for the getservice service. - getservice_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (args.server_flag && (rank == 0)) { - rc = getservice_server_main(args, comm); - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, node 0, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref getservice_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - sleep(args.delay); // give server time to get started - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url.c_str(), args.timeout, &getservice_svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(getservice_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(getservice_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&getservice_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - getservice_client_main(args, getservice_svc, comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if (client_rank == 0) { - log_debug(debug_level, "%d: Halting getservice service", rank); - rc = nssi_kill(&getservice_svc, 0, 5000); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/getservice/getservice_test.h b/packages/trios/tests/getservice/getservice_test.h deleted file mode 100644 index d3c8ae1b3203..000000000000 --- a/packages/trios/tests/getservice/getservice_test.h +++ /dev/null @@ -1,104 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * getservice_client.h - * - * Created on: Jul 20, 2012 - * Author: thkorde - */ - -#ifndef GETSERVICE_CLIENT_H_ -#define GETSERVICE_CLIENT_H_ - -#include -#include - -#include "Trios_logger.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - -#include "getservice_debug.h" - - -/** - * Options and arguments passed to the client driver. - */ -struct getservice_args { - bool client_flag; - bool server_flag; - int transport; - std::string transport_name; - std::string server_url; - std::string url_file; - log_level debug_level; - std::string logfile; - int num_trials; - int num_reqs; - std::string result_file; - std::string result_file_mode; - int timeout; - int delay; - int num_retries; -}; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - - -#else /* K&R C */ -#endif - - - - -#ifdef __cplusplus -} -#endif - - -#endif /* GETSERVICE_CLIENT_H_ */ diff --git a/packages/trios/tests/injection/CMakeLists.txt b/packages/trios/tests/injection/CMakeLists.txt deleted file mode 100644 index ec72324ff979..000000000000 --- a/packages/trios/tests/injection/CMakeLists.txt +++ /dev/null @@ -1,100 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -############# Special exception for rpcgen extra commas in enum ########## -IF (CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") -ENDIF (CMAKE_COMPILER_IS_GNUCXX) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -# We want the binary directory to be a system directory because it contains -# code generated by rpcgen, may have many warnings. -#INCLUDE_DIRECTORIES(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}) - -# Generate the XDR header and source file for the service arguments -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/injection_service_args.x) - - - -TRIBITS_ADD_EXECUTABLE( - InjectionTest - SOURCES injection_service_args.c injection_server.cpp injection_test.cpp injection_client.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -SET(EXTRA_TEST_ARGS "") - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -SET(TESTNAMES - empty-request-sync - empty-request-async) - - -foreach (testname ${TESTNAMES}) - - SET(DEFAULT_ARGS "--io-method=${testname} ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file=${testname}-result.out ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file-mode=a ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-trials=5 ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-reqs=10 ") - - TRIBITS_ADD_TEST( - InjectionTest - NOEXEPREFIX - NAME InjectionTest_${testname} - ARGS "${DEFAULT_ARGS} ${EXTRA_TEST_ARGS}" - COMM serial mpi - NUM_MPI_PROCS 2) -endforeach() diff --git a/packages/trios/tests/injection/cmake/Dependencies.cmake b/packages/trios/tests/injection/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/tests/injection/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/injection/injection_client.cpp b/packages/trios/tests/injection/injection_client.cpp deleted file mode 100644 index aff4cb8b4742..000000000000 --- a/packages/trios/tests/injection/injection_client.cpp +++ /dev/null @@ -1,376 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * injection_client.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -/** - * @defgroup injection_client Data Transfer Client - * - * @ingroup injection_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "injection_test.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - -#include - - -log_level client_debug_level = LOG_UNDEFINED; - -extern int print_args( - std::ostream &out, - const struct injection_args &args, - const char *prefix); - - - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. The server sends the data back via RDMA. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. The server is expected to send the data - * back using nssi_put_data(). - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int injection_empty_request( - const nssi_service *svc, - nssi_request *req) -{ - int rc = NSSI_OK; - log_level debug_level = client_debug_level; - - log_debug(debug_level, "Calling RPC for INJECTION_EMPTY_REQUEST_OP"); - - /* call the remote methods */ - rc = nssi_call_rpc(svc, INJECTION_EMPTY_REQUEST_OP, NULL, NULL, 0, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call INJECTION_EMPTY_REQUEST_OP: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array with the request structure. The server sends the data back in the result. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int injection_empty_request_blk( - const nssi_service *svc) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = injection_empty_request(svc, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - - - -int read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return(-1); - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return(1); - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(client_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); - - return(0); -} - - - - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param injection_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -injection_client_main (struct injection_args &args, nssi_service &injection_svc, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int i,j; - double time; - double start_time; - std::ofstream result_stream; - log_level debug_level = LOG_WARN; - int client_rank, client_size; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - std::vector < nssi_request > reqs(num_reqs); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(INJECTION_EMPTY_REQUEST_OP, void, void, void); - - - log_debug(debug_level, "%d: Starting experiment loop", client_rank); - - /* loop over the experiments (count == num_trials, num_reqs == ops_per_trial) */ - for (i=0; i -#include - -#include -#include -#include -#include -#include -#include - -#include - - -log_level injection_debug_level = LOG_UNDEFINED; - - - -/** - * @brief Emulate a read operation where the bulk data is sent using - * the \ref nssi_put function. - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int injection_empty_request_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const void *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc; - log_level debug_level = injection_debug_level; - - /* process array (nothing to do) */ - log_debug(debug_level, "starting injection_empty_request_srvr"); - - rc = nssi_send_result(caller, request_id, NSSI_OK, NULL, res_addr); - - return rc; -} - - - -void make_progress(bool is_idle) -{ - log_debug(injection_debug_level, "current_time(%llu) is_idle(%llu)", (uint64_t)trios_get_time_ms(), (uint64_t)is_idle); - - return; -} - - -/** - * @brief The NSSI injection-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int injection_server_main(struct injection_args &args, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service injection_svc; - int server_rank; - - MPI_Comm_rank(server_comm, &server_rank); - - /* options that can be overriden by the command-line */ - std::string server_url(NSSI_URL_LEN, '\0'); /* NNTI-style url of the server */ - std::string logfile(""); - - - memset(&injection_svc, 0, sizeof(nssi_service)); - - - /* initialize the nssi service */ - rc = nssi_service_init((nssi_rpc_transport)args.transport, NSSI_SHORT_REQUEST_SIZE, &injection_svc); - if (rc != NSSI_OK) { - log_error(injection_debug_level, "could not init injection_svc: %s", - nssi_err_str(rc)); - return -1; - } - - // register callbacks for the service methods - NSSI_REGISTER_SERVER_STUB(INJECTION_EMPTY_REQUEST_OP, injection_empty_request_srvr, void, void); - - - // Get the Server URL - std::string url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &url[0], NSSI_URL_LEN); - - - // Set the maxumum number of requests to handle (-1 == infinite) - injection_svc.max_reqs = -1; - // injection_svc.progress_callback=(uint64_t)make_progress; - // injection_svc.progress_callback_timeout=100; - - log_debug(injection_debug_level, "Starting Server: url = %s", url.c_str()); - - // Tell the NSSI server to output log data - //rpc_debug_level = injection_debug_level; - - // start processing requests, the client will send a request to exit when done - rc = nssi_service_start(&injection_svc); - if (rc != NSSI_OK) { - log_info(injection_debug_level, "exited injection_svc: %s", - nssi_err_str(rc)); - } - - sleep(5); - - /* shutdown the injection_svc */ - log_debug(injection_debug_level, "shutting down service library"); - nssi_service_fini(&injection_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/injection/injection_service_args.x b/packages/trios/tests/injection/injection_service_args.x deleted file mode 100644 index 8ea098f19fb1..000000000000 --- a/packages/trios/tests/injection/injection_service_args.x +++ /dev/null @@ -1,99 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file injection_service_args.x - * - * @brief Type definitions for an request injection test. - * - * @author Todd kordenbrock (thkorde\@sandia.gov). - * - * Created on: Aug 22, 2011 - * - */ - -/** - * @defgroup injection_example Nessie Data Transfer Example - * - * The data-transfer example demonstrates a simple client - * and server that transfer an array of 16-byte \ref data_t - * data structures from a parallel application to a set of - * servers. We implemented two variations: - * one that transfers the array of data structures - * with the request, and a second method that has each server - * pull the data using the \ref nssi_get_data() function. Although - * this example is fairly simple, it makes a decent benchmark code - * to evaluate overheads of the Nessie transfer protocols and encoding - * schemes. - * -*/ - -/** - * @defgroup injection_types Nessie Example Types - * @ingroup injection_example - * - * @{ - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file. */ -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -/** - * @brief Opcodes for the types of transfer operations. - */ -enum injection_op { - /** Opcode for sending a zero-length request. */ - INJECTION_EMPTY_REQUEST_OP = 1 -}; - -/** - * @} - */ diff --git a/packages/trios/tests/injection/injection_test.cpp b/packages/trios/tests/injection/injection_test.cpp deleted file mode 100644 index 1f4af6f2a177..000000000000 --- a/packages/trios/tests/injection/injection_test.cpp +++ /dev/null @@ -1,560 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * injection_service_test.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include "injection_test.h" -#include "injection_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -int injection_server_main(struct injection_args &args, MPI_Comm server_comm); -int injection_client_main (struct injection_args &args, nssi_service &injection_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct injection_args &args, - const char *prefix) -{ - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url << std::endl; - out << prefix << " \tserver-url-file = " << args.url_file << std::endl; - - if (args.client_flag) { - out << prefix << " \ttransport = " << args.transport_name << std::endl; - out << prefix << " \tio-method = " << args.io_method_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - nssi_service injection_svc; - - int transport_index=-1; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct injection_args args; - - const int num_io_methods = 2; - const int io_method_vals[] = { - INJECTION_EMPTY_REQUEST_SYNC, INJECTION_EMPTY_REQUEST_ASYNC}; - const char * io_method_names[] = { - "empty-request-sync", "empty-request-async"}; - - const int nssi_transport_list[] = { - NSSI_RPC_PTL, - NSSI_RPC_PTL, - NSSI_RPC_IB, - NSSI_RPC_IB, - NSSI_RPC_GEMINI, - NSSI_RPC_GEMINI, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGQPAMI, - NSSI_RPC_BGQPAMI, - NSSI_RPC_MPI}; - - const int num_nssi_transports = 11; - const int nssi_transport_vals[] = { - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10 - }; - const char * nssi_transport_names[] = { - "portals", - "ptl", - "infiniband", - "ib", - "gemini", - "gni", - "bgpdcmf", - "dcmf", - "bgqpami", - "pami", - "mpi" - }; - - - // Initialize arguments - args.transport=NSSI_DEFAULT_TRANSPORT; - args.delay = 1; - args.io_method = INJECTION_EMPTY_REQUEST_SYNC; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.timeout = 500; - args.num_retries = 5; - args.server_url = ""; - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("server", "no-server", &args.server_flag, "Run the server" ); - parser.setOption("client", "no-client", &args.client_flag, "Run the client"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("server-url", &args.server_url, "URL client uses to find the server"); - parser.setOption("server-url-file", &args.url_file, "File that has URL client uses to find server"); - - // Set an enumeration command line option for the io_method - - parser.setOption("io-method", &args.io_method, num_io_methods, io_method_vals, io_method_names, - "I/O Methods for the example: \n" - "\t\t\tempty-request-sync : Send an empty request - synchronous\n" - "\t\t\tempty-request-async: Send an empty request - asynchronous"); - - // Set an enumeration command line option for the NNTI transport - parser.setOption("transport", &transport_index, num_nssi_transports, nssi_transport_vals, nssi_transport_names, - "NSSI transports (not all are available on every platform): \n" - "\t\t\tportals|ptl : Cray or Schutt\n" - "\t\t\tinfiniband|ib : libibverbs\n" - "\t\t\tgemini|gni : Cray libugni (Gemini or Aries)\n" - "\t\t\tbgpdcmf|dcmf : IBM BG/P DCMF\n" - "\t\t\tbgqpami|pami : IBM BG/Q PAMI\n" - "\t\t\tmpi : isend/irecv implementation\n" - ); - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(LOG_ALL, "transport_index=%d", transport_index); - if (transport_index > -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - args.io_method_name=io_method_names[args.io_method]; - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting injection-service test", rank); - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - if (args.client_flag && args.server_flag) { - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - - int color = (rank == 0) ? 0 : 1; // only one server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - // std::cout << "rank=" << rank << "/" << np << ", color=" << color << - // ", new_rank=" << newrank << "/" << newsize << std::endl << std::endl; - // - // std::cout << "my_url=" << my_url << ", server_url=" << args.server_url << std::endl; - } - else { - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - } - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - if (args.server_flag && !args.server_url.empty()) { - // use the server URL as suggested URL - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, args.server_url.c_str()); - } - else { - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - } - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // Broadcast the server URL to all the clients - args.server_url.resize(NSSI_URL_LEN, '\0'); - if (args.server_flag && args.client_flag) { - args.server_url = my_url; - MPI_Bcast(&args.server_url[0], args.server_url.size(), MPI_CHAR, 0, MPI_COMM_WORLD); - } - - else if (!args.server_flag && args.client_flag){ - if (args.server_url.empty()) { - - // check to see if we're supposed to get the URL from a file - if (!args.url_file.empty()) { - // Fetch the server URL from a file - sleep(1); - log_debug(debug_level, "Reading from file %s", args.url_file.c_str()); - std::ifstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - if (urlfile.good()) - getline(urlfile, args.server_url); - } - else { - log_error(debug_level, "Failed to open server_url_file=%s", args.url_file.c_str()); - exit(1); - } - urlfile.close(); - log_debug(debug_level, "URL = %s", args.server_url.c_str()); - } - else { - log_error(debug_level, "Need to set --server-url=[ADDR] or --server-url-file=[PATH]"); - } - } - } - - else if (args.server_flag && !args.client_flag) { - args.server_url = my_url; - - // If the url_file value is set, write the url to a file - if (!args.url_file.empty()) { - std::ofstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - urlfile << args.server_url.c_str() << std::endl; - } - urlfile.close(); - log_debug(debug_level, "Wrote url to file %s", args.url_file.c_str()); - } - } - - - - // Set the debug level for the injection service. - injection_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (args.server_flag && (rank == 0)) { - rc = injection_server_main(args, comm); - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, node 0, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref injection_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - sleep(args.delay); // give server time to get started - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url.c_str(), args.timeout, &injection_svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(injection_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(injection_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&injection_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - injection_client_main(args, injection_svc, comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if (client_rank == 0) { - log_debug(debug_level, "%d: Halting injection service", rank); - rc = nssi_kill(&injection_svc, 0, 5000); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/injection/injection_test.h b/packages/trios/tests/injection/injection_test.h deleted file mode 100644 index 50758174638b..000000000000 --- a/packages/trios/tests/injection/injection_test.h +++ /dev/null @@ -1,111 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * injection_client.h - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -#ifndef INJECTION_CLIENT_H_ -#define INJECTION_CLIENT_H_ - -#include -#include - -#include "Trios_logger.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - - - -enum IO_METHODS { - INJECTION_EMPTY_REQUEST_SYNC, - INJECTION_EMPTY_REQUEST_ASYNC -}; - - -/** - * Options and arguments passed to the client driver. - */ -struct injection_args { - bool client_flag; - bool server_flag; - int transport; - std::string transport_name; - int io_method; - std::string server_url; - std::string url_file; - std::string io_method_name; - log_level debug_level; - std::string logfile; - int num_trials; - int num_reqs; - std::string result_file; - std::string result_file_mode; - int timeout; - int delay; - int num_retries; -}; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - - -#else /* K&R C */ -#endif - - - - -#ifdef __cplusplus -} -#endif - - -#endif /* INJECTION_CLIENT_H_ */ diff --git a/packages/trios/tests/multicast/CMakeLists.txt b/packages/trios/tests/multicast/CMakeLists.txt deleted file mode 100644 index fc0d4aef7bda..000000000000 --- a/packages/trios/tests/multicast/CMakeLists.txt +++ /dev/null @@ -1,104 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -############# Special exception for rpcgen extra commas in enum ########## -IF (CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") -ENDIF (CMAKE_COMPILER_IS_GNUCXX) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -# We want the binary directory to be a system directory because it contains -# code generated by rpcgen, may have many warnings. -#INCLUDE_DIRECTORIES(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}) - -# Generate the XDR header and source file for the service arguments -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/multicast_service_args.x) - - - -TRIBITS_ADD_EXECUTABLE( - MulticastTest - SOURCES multicast_service_args.c multicast_server.cpp multicast_test.cpp multicast_client.cpp multicast_buffer_util.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -SET(EXTRA_TEST_ARGS "") - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -SET(TESTNAMES - empty-request-sync - empty-request-async - get-sync - get-async - put-sync - put-async) - - -foreach (testname ${TESTNAMES}) - - SET(DEFAULT_ARGS "--io-method=${testname} ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file=${testname}-result.out ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file-mode=a ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-trials=5 ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-reqs=10 ") - - TRIBITS_ADD_TEST( - MulticastTest - NOEXEPREFIX - NAME MulticastTest_${testname} - ARGS "${DEFAULT_ARGS} ${EXTRA_TEST_ARGS}" - COMM serial mpi - NUM_MPI_PROCS 3) -endforeach() diff --git a/packages/trios/tests/multicast/cmake/Dependencies.cmake b/packages/trios/tests/multicast/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/tests/multicast/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/multicast/multicast_buffer_util.cpp b/packages/trios/tests/multicast/multicast_buffer_util.cpp deleted file mode 100644 index e5b1a9d00fab..000000000000 --- a/packages/trios/tests/multicast/multicast_buffer_util.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -/* - * multicast_init_buffer.cpp - * - * Created on: Nov 4, 2011 - * Author: raoldfi - */ - - -#include "multicast_service_args.h" -#include "multicast_debug.h" -#include - - -/** - * Initialize an array given a starting seed. - * - * This function is used by both the client and the server to generate - * initialize values in a buffer. - */ -void multicast_init_data_array(const unsigned int seed, data_array_t *array) -{ - //char state[8]; - - int len = array->data_array_t_len; - data_t *buf = array->data_array_t_val; - - /* initialize the random seed */ - //initstate(seed, state, 8); - - for (int i=0; idata_array_t_len != arr2->data_array_t_len) { - log_error(debug_level, "arr1->len=%d, arr2->len=%d", - arr1->data_array_t_len, arr2->data_array_t_len); - return -1; - } - - for (int i=0; i<(int)arr1->data_array_t_len; i++) { - if (arr1->data_array_t_val[i].int_val != arr2->data_array_t_val[i].int_val) { - log_error(debug_level, "val[%d].int_val=%d, val[%d].int_val=%d", - i,arr1->data_array_t_val[i].int_val, - i,arr2->data_array_t_val[i].int_val); - return -1; - } - if (arr1->data_array_t_val[i].float_val != arr2->data_array_t_val[i].float_val) { - log_error(debug_level, "val[%d].float_val=%f, val[%d].float_val=%f", - i,arr1->data_array_t_val[i].float_val, - i,arr2->data_array_t_val[i].float_val); - return -1; - } - if (arr1->data_array_t_val[i].double_val != arr2->data_array_t_val[i].double_val) { - log_error(debug_level, "val[%d].double_val=%g, val[%d].double_val=%g", - i,arr1->data_array_t_val[i].double_val, - i,arr2->data_array_t_val[i].double_val); - return -1; - } - } - - return 0; - -} - - -int multicast_validate_array(const int seed, data_array_t *array) -{ - int rc = 0; - log_level debug_level = multicast_debug_level; - - /* Validate the array that was sent through the args */ - data_array_t tmp_array; - - tmp_array.data_array_t_len = array->data_array_t_len; - tmp_array.data_array_t_val = new data_t[array->data_array_t_len]; - - multicast_init_data_array(seed, &tmp_array); - rc = multicast_compare_data_arrays(&tmp_array, array); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - - delete [] tmp_array.data_array_t_val; - return rc; -} diff --git a/packages/trios/tests/multicast/multicast_client.cpp b/packages/trios/tests/multicast/multicast_client.cpp deleted file mode 100644 index 4752ad873b0b..000000000000 --- a/packages/trios/tests/multicast/multicast_client.cpp +++ /dev/null @@ -1,674 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * multicast_client.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -/** - * @defgroup multicast_client Data Transfer Client - * - * @ingroup multicast_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "multicast_test.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - -#include - - -log_level client_debug_level = LOG_UNDEFINED; - -/* prototype for a function to initialize buffers */ -extern void multicast_init_data_array(const unsigned int seed, data_array_t *array); -extern int multicast_compare_data_arrays(const data_array_t *arr1, const data_array_t *arr2); - -extern int print_args( - std::ostream &out, - const struct multicast_args &args, - const char *prefix); - - - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. The server sends the data back via RDMA. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. The server is expected to send the data - * back using nssi_put_data(). - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int multicast_empty_request( - const nssi_service *svc, - nssi_request *req) -{ - int rc = NSSI_OK; - log_level debug_level = client_debug_level; - - log_debug(debug_level, "Calling RPC for MULTICAST_EMPTY_REQUEST_OP"); - - /* call the remote methods */ - rc = nssi_multicast_rpc(svc, 2, MULTICAST_EMPTY_REQUEST_OP, NULL, NULL, 0, NULL, 0, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call MULTICAST_EMPTY_REQUEST_OP: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array with the request structure. The server sends the data back in the result. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int multicast_empty_request_blk( - const nssi_service *svc) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req[2]; - - /* call the async function */ - rc = multicast_empty_request(svc, &req[0]); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_waitall(&req[0], 2, -1); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - -int multicast_get( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array, - nssi_request *req) -{ - int rc = NSSI_OK; - log_level debug_level = client_debug_level; - multicast_get_args args; - int nbytes; - - /* the buffer to send to the server */ - const data_t *buf = array->data_array_t_val; - - log_debug(debug_level, "Calling RPC for MULTICAST_GET_OP"); - - /* initialize the arguments */ - memset(&args, 0, sizeof(multicast_get_args)); - - /* the only argument to send is the size of the array */ - args.len = len; - args.seed = seed; - args.validate = validate; - - /* calculate the number of bytes in the buffer */ - nbytes = args.len*sizeof(data_t); - - /* call the remote methods */ - rc = nssi_multicast_rpc(svc, 2, MULTICAST_GET_OP, &args, (char *)buf, nbytes, NULL, 0, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call MULTICAST_GET_OP: %s", - nssi_err_str(rc)); - } - - return rc; -} - -int multicast_get_blk( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req[2]; - - /* call the async function */ - rc = multicast_get(svc, len, seed, validate, array, &req[0]); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_waitall(&req[0], 2, -1); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; -} - -int multicast_put( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - data_array_t *put_buf, - nssi_request *req) -{ - int rc = NSSI_OK; - multicast_put_args args; - log_level debug_level = client_debug_level; - - int nbytes; - - log_debug(debug_level, "Calling RPC for MULTICAST_GET_OP"); - - /* initialize the arguments */ - memset(&args, 0, sizeof(multicast_put_args)); - - assert(len == put_buf->data_array_t_len); - - /* tell the server how about the buffer */ - args.len = len; - args.seed = seed; - args.validate = validate; - - /* calculate the number of bytes in the buffer */ - nbytes = args.len*sizeof(data_t); - - data_t *buf; - //buf = (data_t *)malloc(nbytes); - buf = put_buf->data_array_t_val; - - /* call the remote methods */ - rc = nssi_multicast_rpc(svc, 2, MULTICAST_PUT_OP, &args, buf, nbytes, NULL, 0, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call MULTICAST_GET_OP: %s", - nssi_err_str(rc)); - } - - return rc; -} - -int multicast_put_blk( - const nssi_service *svc, - const uint32_t len, - const uint32_t seed, - const bool validate, - data_array_t *put_buf) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req[2]; - - /* call the async function */ - rc = multicast_put(svc, len, seed, validate, put_buf, &req[0]); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_waitall(&req[0], 2, -1); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; -} - - - -int read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return(-1); - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return(1); - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(client_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); - - return(0); -} - - - - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param multicast_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -multicast_client_main (struct multicast_args &args, nssi_service* multicast_svc, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int i,j; - double time; - double start_time; - std::ofstream result_stream; - log_level debug_level = LOG_WARN; - int client_rank, client_size; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - nssi_request *reqs = (nssi_request *)malloc(2*num_reqs*sizeof(nssi_request)); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(MULTICAST_EMPTY_REQUEST_OP, void, void, void); - NSSI_REGISTER_CLIENT_STUB(MULTICAST_GET_OP, multicast_get_args, void, void); - NSSI_REGISTER_CLIENT_STUB(MULTICAST_PUT_OP, multicast_put_args, void, void); - - /* allocate space for the data arrays used for testing */ - std::vector array_vec; - array_vec.reserve(args.num_reqs); // async jobs need num_reqs arrays - - for (i=0; i -#include - -#include -#include -#include -#include -#include -#include - -#include - - -log_level multicast_debug_level = LOG_UNDEFINED; - - -/* prototype for a function to initialize buffers */ -extern void multicast_init_data_array(const unsigned int seed, data_array_t *array); -extern int multicast_compare_data_arrays(const data_array_t *arr1, const data_array_t *arr2); - -/** - * @brief Emulate a read operation where the bulk data is sent using - * the \ref nssi_put function. - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int multicast_empty_request_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const void *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc; - log_level debug_level = multicast_debug_level; - - /* process array (nothing to do) */ - log_debug(debug_level, "starting multicast_empty_request_srvr"); - - rc = nssi_send_result(caller, request_id, NSSI_OK, NULL, res_addr); - - return rc; -} - -int multicast_get_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const multicast_get_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc; - log_level debug_level = multicast_debug_level; - - const int len = args->len; - const long int seed = args->seed; - const bool validate = args->validate; - - int nbytes = len*sizeof(data_t); - - /* process array (nothing to do) */ - log_debug(debug_level, "starting multicast_get_srvr"); - - /* allocate space for the incoming buffer */ - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = (data_t *)malloc(nbytes); - - log_debug(debug_level, "getting data from client (%s)", caller->url); - - /* now we need to fetch the data from the client */ - rc = nssi_get_data(caller, array.data_array_t_val, nbytes, data_addr); - if (rc != NSSI_OK) { - log_warn(debug_level, "could not fetch data from client"); - return rc; - } - - /* Validate the array */ - if (validate && (rc == 0)) { - data_array_t tmp_array; - tmp_array.data_array_t_len = len; - tmp_array.data_array_t_val = (data_t *)malloc(nbytes); - - multicast_init_data_array(seed, &tmp_array); - rc = multicast_compare_data_arrays(&tmp_array, &array); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - - free(tmp_array.data_array_t_val); - } - - rc = nssi_send_result(caller, request_id, NSSI_OK, NULL, res_addr); - - return rc; -} - -int multicast_put_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const multicast_put_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc; - log_level debug_level = multicast_debug_level; - - const bool validate = args->validate; - const int seed = args->seed; - - int nbytes = args->len*sizeof(data_t); - - /* process array (nothing to do) */ - log_debug(debug_level, "starting multicast_put_srvr"); - - /* allocate space for the outgoing buffer */ - data_array_t array; - array.data_array_t_len = args->len; - array.data_array_t_val = (data_t *)malloc(nbytes); - - /* if we need to validate the array, it needs to be initialized */ - if (validate) { - multicast_init_data_array(seed, &array); - } - - /* now we need to put the data to the client */ - rc = nssi_put_data(caller, array.data_array_t_val, nbytes, data_addr, -1); - if (rc != NSSI_OK) { - log_warn(debug_level, "could not put data to client"); - return rc; - } - - rc = nssi_send_result(caller, request_id, NSSI_OK, NULL, res_addr); - - return rc; -} - - - -void make_progress(bool is_idle) -{ - log_debug(multicast_debug_level, "current_time(%llu) is_idle(%llu)", (uint64_t)trios_get_time_ms(), (uint64_t)is_idle); - - return; -} - - -/** - * @brief The NSSI multicast-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int multicast_server_main(struct multicast_args &args, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service multicast_svc; - int server_rank; - - MPI_Comm_rank(server_comm, &server_rank); - - /* options that can be overriden by the command-line */ - std::string server_url(NSSI_URL_LEN, '\0'); /* NNTI-style url of the server */ - std::string logfile(""); - - - memset(&multicast_svc, 0, sizeof(nssi_service)); - - - /* initialize the nssi service */ - rc = nssi_service_init((nssi_rpc_transport)args.transport, NSSI_SHORT_REQUEST_SIZE, &multicast_svc); - if (rc != NSSI_OK) { - log_error(multicast_debug_level, "could not init multicast_svc: %s", - nssi_err_str(rc)); - return -1; - } - - // register callbacks for the service methods - NSSI_REGISTER_SERVER_STUB(MULTICAST_EMPTY_REQUEST_OP, multicast_empty_request_srvr, void, void); - NSSI_REGISTER_SERVER_STUB(MULTICAST_GET_OP, multicast_get_srvr, multicast_get_args, void); - NSSI_REGISTER_SERVER_STUB(MULTICAST_PUT_OP, multicast_put_srvr, multicast_put_args, void); - - - // Get the Server URL - std::string url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &url[0], NSSI_URL_LEN); - - - // Set the maxumum number of requests to handle (-1 == infinite) - multicast_svc.max_reqs = -1; - // multicast_svc.progress_callback=(uint64_t)make_progress; - // multicast_svc.progress_callback_timeout=100; - - log_debug(multicast_debug_level, "Starting Server: url = %s", url.c_str()); - - // Tell the NSSI server to output log data - //rpc_debug_level = multicast_debug_level; - - // start processing requests, the client will send a request to exit when done - rc = nssi_service_start(&multicast_svc); - if (rc != NSSI_OK) { - log_info(multicast_debug_level, "exited multicast_svc: %s", - nssi_err_str(rc)); - } - - sleep(5); - - /* shutdown the multicast_svc */ - log_debug(multicast_debug_level, "shutting down service library"); - nssi_service_fini(&multicast_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/multicast/multicast_service_args.x b/packages/trios/tests/multicast/multicast_service_args.x deleted file mode 100644 index deb425edb612..000000000000 --- a/packages/trios/tests/multicast/multicast_service_args.x +++ /dev/null @@ -1,159 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -/** - * @file multicast_service_args.x - * - * @brief Type definitions for an request multicast test. - * - * @author Todd kordenbrock (thkorde\@sandia.gov). - * - * Created on: Aug 22, 2011 - * - */ - -/** - * @defgroup multicast_example Nessie Data Transfer Example - * - * The data-transfer example demonstrates a simple client - * and server that transfer an array of 16-byte \ref data_t - * data structures from a parallel application to a set of - * servers. We implemented two variations: - * one that transfers the array of data structures - * with the request, and a second method that has each server - * pull the data using the \ref nssi_get_data() function. Although - * this example is fairly simple, it makes a decent benchmark code - * to evaluate overheads of the Nessie transfer protocols and encoding - * schemes. - * -*/ - -/** - * @defgroup multicast_types Nessie Example Types - * @ingroup multicast_example - * - * @{ - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file. */ -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -/** - * @brief Opcodes for the types of transfer operations. - */ -enum multicast_op { - /** Opcode for sending a zero-length request. */ - MULTICAST_EMPTY_REQUEST_OP = 1, - MULTICAST_GET_OP = 2, - MULTICAST_PUT_OP = 3 -}; - -/** - * @brief A 16-byte structure that contains an int, float, and double. - * - * This structure contains an int, float, and double as an example - * of a complex structure with multiple types. This will exercise the - * encoding/decoding features of Nessie. - */ -struct data_t { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -}; - -/** - * @brief Array of 16-byte structures that we can send with the request. - * - * Rpcgen will use this definition to define encoding functions to - * encode and decode an array of \ref data_t structures. We will - * use these functions when sending the array with the request. - */ -typedef data_t data_array_t<>; - -/** - * @brief Arguments for the second transfer operation, MULTICAST_GET_OP. - * - * The second transfer operation only needs to send the length - * of the data array. It uses the data argument in the nssi_call_rpc() - * to identify the raw data for the server to fetch. - */ -struct multicast_get_args { - /** The length of the data array. */ - int32_t len; - /** A seed for initializing the array */ - uint32_t seed; - /** A flag to perform validation */ - bool validate; -}; - -/** - * @brief Arguments for the third transfer operation, MULTICAST_PUT_OP. - * - * The third transfer operation only needs to send the length - * of the data array. It uses the data argument in the nssi_call_rpc() - * to identify the destination of the raw data. - */ -struct multicast_put_args { - /** The length of the data array. */ - int32_t len; - /** A seed for initializing the array */ - uint32_t seed; - /** A flag to perform validation */ - bool validate; -}; - -/** - * @} - */ diff --git a/packages/trios/tests/multicast/multicast_test.cpp b/packages/trios/tests/multicast/multicast_test.cpp deleted file mode 100644 index b4840ceb2ef1..000000000000 --- a/packages/trios/tests/multicast/multicast_test.cpp +++ /dev/null @@ -1,627 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * multicast_service_test.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include "multicast_test.h" -#include "multicast_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -int multicast_server_main(struct multicast_args &args, MPI_Comm server_comm); -int multicast_client_main (struct multicast_args &args, nssi_service* multicast_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct multicast_args &args, - const char *prefix) -{ - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url << std::endl; - - if (args.client_flag) { - out << prefix << " \ttransport = " << args.transport_name << std::endl; - out << prefix << " \tio-method = " << args.io_method_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tlen = " << args.len << std::endl; - out << prefix << " \tvalidate = " << ((args.validate_flag)?"true":"false") << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - nssi_service multicast_svc[2]; - - int transport_index=-1; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct multicast_args args; - - const int num_io_methods = 6; - const int io_method_vals[] = { - MULTICAST_EMPTY_REQUEST_SYNC, MULTICAST_EMPTY_REQUEST_ASYNC, - MULTICAST_GET_SYNC, MULTICAST_GET_ASYNC, - MULTICAST_PUT_SYNC, MULTICAST_PUT_ASYNC}; - const char * io_method_names[] = { - "empty-request-sync", "empty-request-async", - "get-sync", "get-async", - "put-sync", "put-async"}; - - const int nssi_transport_list[] = { - NSSI_RPC_PTL, - NSSI_RPC_PTL, - NSSI_RPC_IB, - NSSI_RPC_IB, - NSSI_RPC_GEMINI, - NSSI_RPC_GEMINI, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGQPAMI, - NSSI_RPC_BGQPAMI, - NSSI_RPC_MPI}; - - const int num_nssi_transports = 11; - const int nssi_transport_vals[] = { - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10 - }; - const char * nssi_transport_names[] = { - "portals", - "ptl", - "infiniband", - "ib", - "gemini", - "gni", - "bgpdcmf", - "dcmf", - "bgqpami", - "pami", - "mpi" - }; - - - // Initialize arguments - args.transport=NSSI_DEFAULT_TRANSPORT; - args.delay = 1; - args.io_method = MULTICAST_EMPTY_REQUEST_SYNC; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.len = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file[0] = ""; - args.url_file[1] = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.timeout = 500; - args.num_retries = 5; - args.validate_flag = true; - args.server_url[0] = ""; - args.server_url[1] = ""; - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("server", "no-server", &args.server_flag, "Run the server" ); - parser.setOption("client", "no-client", &args.client_flag, "Run the client"); - parser.setOption("len", &args.len, "The number of structures in an input buffer"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("server-url-1", &args.server_url[0], "URL client uses to find the server 1"); - parser.setOption("server-url-2", &args.server_url[1], "URL client uses to find the server 2"); - parser.setOption("server-url-file-1", &args.url_file[0], "File that has URL client uses to find server 1"); - parser.setOption("server-url-file-2", &args.url_file[1], "File that has URL client uses to find server 2"); - parser.setOption("validate", "no-validate", &args.validate_flag, "Validate the data"); - - // Set an enumeration command line option for the io_method - - parser.setOption("io-method", &args.io_method, num_io_methods, io_method_vals, io_method_names, - "I/O Methods for the example: \n" - "\t\t\tempty-request-sync : Send an empty request - synchronous\n" - "\t\t\tempty-request-async: Send an empty request - asynchronous\n" - "\t\t\tget-sync : Servers pull data from client - synchronous\n" - "\t\t\tget-async: Servers pull data from client - asynchronous\n" - "\t\t\tput-sync : Servers push data from client - synchronous\n" - "\t\t\tput-async: Servers push data from client - asynchronous" - ); - - // Set an enumeration command line option for the NNTI transport - parser.setOption("transport", &transport_index, num_nssi_transports, nssi_transport_vals, nssi_transport_names, - "NSSI transports (not all are available on every platform): \n" - "\t\t\tportals|ptl : Cray or Schutt\n" - "\t\t\tinfiniband|ib : libibverbs\n" - "\t\t\tgemini|gni : Cray libugni (Gemini or Aries)\n" - "\t\t\tbgpdcmf|dcmf : IBM BG/P DCMF\n" - "\t\t\tbgqpami|pami : IBM BG/Q PAMI\n" - "\t\t\tmpi : isend/irecv implementation\n" - ); - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(LOG_ALL, "transport_index=%d", transport_index); - if (transport_index > -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - args.io_method_name=io_method_names[args.io_method]; - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting multicast-service test", rank); - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - if (args.client_flag && args.server_flag) { - if (np < 3) { - log_error(debug_level, "Must use at least 3 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - - int color = ((rank == 0)||(rank == 1)) ? 0 : 1; // two server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - // std::cout << "rank=" << rank << "/" << np << ", color=" << color << - // ", new_rank=" << newrank << "/" << newsize << std::endl << std::endl; - // - // std::cout << "my_url=" << my_url << ", server_url=" << args.server_url << std::endl; - } - else { - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - } - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - if (args.server_flag && !args.server_url[rank].empty()) { - // use the server URL as suggested URL - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, args.server_url[rank].c_str()); - } - else { - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - } - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // Broadcast the server URL to all the clients - args.server_url[0].resize(NSSI_URL_LEN, '\0'); - args.server_url[1].resize(NSSI_URL_LEN, '\0'); - if (args.server_flag && args.client_flag) { - args.server_url[0] = my_url; - MPI_Bcast(&args.server_url[0][0], args.server_url[0].size(), MPI_CHAR, 0, MPI_COMM_WORLD); - args.server_url[1] = my_url; - MPI_Bcast(&args.server_url[1][0], args.server_url[1].size(), MPI_CHAR, 1, MPI_COMM_WORLD); - } - - else if (!args.server_flag && args.client_flag){ - if (args.server_url[0].empty()) { - - // check to see if we're supposed to get the URL from a file - if (!args.url_file[0].empty()) { - // Fetch the server URL from a file - sleep(1); - log_debug(debug_level, "Reading from file %s", args.url_file[0].c_str()); - std::ifstream urlfile (args.url_file[0].c_str()); - if (urlfile.is_open()) { - if (urlfile.good()) - getline(urlfile, args.server_url[0]); - } - else { - log_error(debug_level, "Failed to open server_url_file=%s", args.url_file[0].c_str()); - exit(1); - } - urlfile.close(); - log_debug(debug_level, "URL = %s", args.server_url[0].c_str()); - } - else { - log_error(debug_level, "Need to set --server-url-1=[ADDR] or --server-url-file-1=[PATH]"); - } - } - if (args.server_url[1].empty()) { - - // check to see if we're supposed to get the URL from a file - if (!args.url_file[1].empty()) { - // Fetch the server URL from a file - sleep(1); - log_debug(debug_level, "Reading from file %s", args.url_file[1].c_str()); - std::ifstream urlfile (args.url_file[1].c_str()); - if (urlfile.is_open()) { - if (urlfile.good()) - getline(urlfile, args.server_url[1]); - } - else { - log_error(debug_level, "Failed to open server_url_file=%s", args.url_file[1].c_str()); - exit(1); - } - urlfile.close(); - log_debug(debug_level, "URL = %s", args.server_url[1].c_str()); - } - else { - log_error(debug_level, "Need to set --server-url-1=[ADDR] or --server-url-file-1=[PATH]"); - } - } - } - - else if (args.server_flag && !args.client_flag) { - args.server_url[0] = my_url; - // If the url_file value is set, write the url to a file - if (!args.url_file[0].empty()) { - std::ofstream urlfile (args.url_file[0].c_str()); - if (urlfile.is_open()) { - urlfile << args.server_url[0].c_str() << std::endl; - } - urlfile.close(); - log_debug(debug_level, "Wrote url to file %s", args.url_file[0].c_str()); - } - - args.server_url[1] = my_url; - // If the url_file value is set, write the url to a file - if (!args.url_file[1].empty()) { - std::ofstream urlfile (args.url_file[1].c_str()); - if (urlfile.is_open()) { - urlfile << args.server_url[1].c_str() << std::endl; - } - urlfile.close(); - log_debug(debug_level, "Wrote url to file %s", args.url_file[1].c_str()); - } - } - - - - // Set the debug level for the multicast service. - multicast_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on nodes 0 and 1. - * In this example, the server is two process. - */ - if (args.server_flag && ((rank == 0)|(rank == 1))) { - rc = multicast_server_main(args, comm); - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, nodes 0 and 1, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref multicast_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - sleep(args.delay); // give server time to get started - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url[0].c_str(), args.timeout, &multicast_svc[0]); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(multicast_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url[1].c_str(), args.timeout, &multicast_svc[1]); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(multicast_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(multicast_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&multicast_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - multicast_client_main(args, &multicast_svc[0], comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if (client_rank == 0) { - log_debug(debug_level, "%d: Halting multicast service", rank); - rc = nssi_kill(&multicast_svc[0], 0, 5000); - rc = nssi_kill(&multicast_svc[1], 0, 5000); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/multicast/multicast_test.h b/packages/trios/tests/multicast/multicast_test.h deleted file mode 100644 index a445c5383621..000000000000 --- a/packages/trios/tests/multicast/multicast_test.h +++ /dev/null @@ -1,117 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * multicast_client.h - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -#ifndef MULTICAST_TEST_H_ -#define MULTICAST_TEST_H_ - -#include -#include - -#include "Trios_logger.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - - - -enum IO_METHODS { - MULTICAST_EMPTY_REQUEST_SYNC, - MULTICAST_EMPTY_REQUEST_ASYNC, - MULTICAST_GET_SYNC, - MULTICAST_GET_ASYNC, - MULTICAST_PUT_SYNC, - MULTICAST_PUT_ASYNC -}; - - -/** - * Options and arguments passed to the client driver. - */ -struct multicast_args { - bool client_flag; - bool server_flag; - int transport; - std::string transport_name; - int len; - int io_method; - std::string server_url[2]; - std::string url_file[2]; - std::string io_method_name; - log_level debug_level; - std::string logfile; - int num_trials; - int num_reqs; - std::string result_file; - std::string result_file_mode; - int timeout; - int delay; - int num_retries; - bool validate_flag; -}; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - - -#else /* K&R C */ -#endif - - - - -#ifdef __cplusplus -} -#endif - - -#endif /* MULTICAST_TEST_H_ */ diff --git a/packages/trios/tests/netperf/CMakeLists.txt b/packages/trios/tests/netperf/CMakeLists.txt deleted file mode 100644 index adcece22dfcb..000000000000 --- a/packages/trios/tests/netperf/CMakeLists.txt +++ /dev/null @@ -1,102 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - - - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - -# Generate the XDR header and source file for the service arguments -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/netperf_service_args.x) - - -#TRIBITS_ADD_EXECUTABLE( -# NetperfMPITest -# SOURCES netperf_service_args.c netperf_util.cpp netperf_mpi_server.cpp netperf_mpi_test.cpp netperf_mpi_client.cpp -# DEPLIBS ${DEPLIBS} -# NOEXEPREFIX -#) - - -TRIBITS_ADD_EXECUTABLE( - NetperfServiceTest - SOURCES netperf_service_args.c netperf_util.cpp netperf_server.cpp netperf_threads.cpp netperf_service_test.cpp netperf_client.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -SET(EXTRA_TEST_ARGS "") - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -SET(TESTNAMES - write-rdma-sync - write-rdma-async - read-rdma-sync - read-rdma-async) - - -foreach (testname ${TESTNAMES}) - - SET(DEFAULT_ARGS "--io-method=${testname} --validate ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file=${testname}-result.out ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --result-file-mode=a ") - SET(DEFAULT_ARGS "${DEFAULT_ARGS} --num-trials=5 ") - - TRIBITS_ADD_TEST( - NetperfServiceTest - NOEXEPREFIX - NAME NetperfService_${testname} - ARGS "${DEFAULT_ARGS} ${EXTRA_TEST_ARGS}" - COMM serial mpi - NUM_MPI_PROCS 2) - - #SET_TESTS_PROPERTIES(${PACKAGE_NAME}_NetperfService_${testname}_MPI_2 PROPERTIES TIMEOUT 10) -endforeach() - - diff --git a/packages/trios/tests/netperf/cmake/Dependencies.cmake b/packages/trios/tests/netperf/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/tests/netperf/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/netperf/netperf_client.cpp b/packages/trios/tests/netperf/netperf_client.cpp deleted file mode 100644 index f48b9b4e1f81..000000000000 --- a/packages/trios/tests/netperf/netperf_client.cpp +++ /dev/null @@ -1,709 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file netperf-client.cpp - * - */ - -/** - * @defgroup netperf_client Data Transfer Client - * - * @ingroup netperf_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "netperf_client.h" - - -#include -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - -#include "netperf_service_args.h" -#include "netperf_util.h" - - - -log_level client_debug_level = LOG_UNDEFINED; - -/* prototype for a function to initialize buffers */ -extern int print_args( - std::ostream &out, - const struct netperf_args &args, - const char *prefix); - -struct netperf_experiment_data { - uint32_t num_reqs; - int32_t array_len; - bool validate; - data_array_t *transfer_array; - data_array_t *validation_array; -}; -typedef struct netperf_experiment_data netperf_experiment_data_t; - -static netperf_experiment_data_t experiment_data; - - -/** - * @brief Asynchronous transfer of data via RDMA. - * - * This method sends the length of the data array to the server. After - * receiving the request, the server pulls the data from the memory - * buffer using the nssi_get_data() function. This function is asynchronous -- - * after sending the request to the server, this function returns the - * request data structure. The caller must at some point call the - * nssi_wait() or nssi_test() function to find out if the request is - * complete. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The service request data structure. - */ -int netperf_write_rdma( - const nssi_service *svc, - const uint32_t buffer_index, - nssi_request *req) -{ - int rc = NSSI_OK; - netperf_write_rdma_args args; - int nbytes=experiment_data.array_len*sizeof(data_t); - - /* the buffer to send to the server */ - const data_t *buf = experiment_data.transfer_array[buffer_index].data_array_t_val; - - /* initialize the arguments */ - memset(&args, 0, sizeof(netperf_write_rdma_args)); - - /* the only argument to send is the size of the array */ - args.buffer_index=buffer_index; - - /* call the remote methods (send buffer using the data portal) */ - rc = nssi_call_rpc(svc, NETPERF_WRITE_RDMA_OP, &args, (char *)buf, nbytes, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call netperf_write_encode: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of data via RDMA. - * - * This method sends the length of the data array to the server. After - * receiving the request, the server pulls the data from the memory - * buffer using the nssi_get_data() function. This function is blocking -- - * it calls the nssi_wait() function to make sure the server completed - * the request before returning. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int netperf_write_rdma_blk( - const nssi_service *svc, - const uint32_t buffer_index) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = netperf_write_rdma(svc, buffer_index, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. The server sends the data back via RDMA. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. The server is expected to send the data - * back using nssi_put_data(). - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int netperf_read_rdma( - const nssi_service *svc, - const uint32_t buffer_index, - nssi_request *req) -{ - int rc = NSSI_OK; - netperf_read_rdma_args args; - log_level debug_level = client_debug_level; - - int nbytes=experiment_data.array_len*sizeof(data_t); - - /* initialize the arguments */ - memset(&args, 0, sizeof(netperf_read_rdma_args)); - - /* tell the server how about the buffer */ - args.buffer_index=buffer_index; - - data_t *buf=experiment_data.transfer_array[buffer_index].data_array_t_val; - - log_debug(debug_level, "Calling RPC for NETPERF_READ_RDMA_OP, len=%d, val=%p, nbytes=%d", - experiment_data.transfer_array[buffer_index].data_array_t_len, buf, nbytes); - - /* call the remote methods */ - rc = nssi_call_rpc(svc, NETPERF_READ_RDMA_OP, &args, buf, nbytes, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call netperf_5: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array with the request structure. The server sends the data back in the result. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int netperf_read_rdma_blk( - const nssi_service *svc, - const uint32_t buffer_index) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = netperf_read_rdma(svc, buffer_index, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - - -int netperf_send_setup( - const nssi_service *svc) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - netperf_experiment_setup_args args; - log_level debug_level = client_debug_level; - - /* initialize the arguments */ - memset(&args, 0, sizeof(netperf_read_rdma_args)); - args.num_reqs = experiment_data.num_reqs; - args.array_len = experiment_data.array_len; - args.validate = experiment_data.validate; - - log_debug(debug_level, "Calling RPC for NETPERF_EXPERIMENT_SETUP_OP, num_reqs=%d, array_len=%p, validate=%d", - experiment_data.num_reqs, experiment_data.array_len, experiment_data.validate); - - /* call the remote methods */ - rc = nssi_call_rpc(svc, NETPERF_EXPERIMENT_SETUP_OP, &args, NULL, 0, NULL, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call netperf_5: %s", - nssi_err_str(rc)); - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - -int netperf_send_teardown( - const nssi_service *svc) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - log_level debug_level = client_debug_level; - - log_debug(debug_level, "Calling RPC for NETPERF_EXPERIMENT_TEARDOWN_OP"); - - /* call the remote methods */ - rc = nssi_call_rpc(svc, NETPERF_EXPERIMENT_TEARDOWN_OP, NULL, NULL, 0, NULL, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call netperf_5: %s", - nssi_err_str(rc)); - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - -int netperf_experiment_setup( - const uint32_t num_reqs, - const int32_t array_len, - const bool validate) -{ - int rc = NSSI_OK; - log_level debug_level = client_debug_level; - - experiment_data.num_reqs = num_reqs; - experiment_data.array_len = array_len; - experiment_data.validate = validate; - - int nbytes = experiment_data.array_len*sizeof(data_t); - - log_debug(debug_level, "starting netperf_experiment_setup"); - - /* allocate and initialize space for the transfer buffers */ - experiment_data.transfer_array=(data_array_t *)malloc(experiment_data.num_reqs*sizeof(data_array_t)); - for (uint32_t i=0; i results; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - std::vector < nssi_request > reqs(num_reqs); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(NETPERF_EXPERIMENT_SETUP_OP, netperf_experiment_setup_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NETPERF_EXPERIMENT_TEARDOWN_OP, void, void, void); - NSSI_REGISTER_CLIENT_STUB(NETPERF_WRITE_RDMA_OP, netperf_write_rdma_args, void, void); - NSSI_REGISTER_CLIENT_STUB(NETPERF_READ_RDMA_OP, netperf_read_rdma_args, void, void); - - netperf_experiment_setup(args.num_reqs, args.len, args.validate_flag); - netperf_send_setup(&netperf_svc); - - log_debug(debug_level, "%d: Starting experiment loop", client_rank); - - /* loop over the experiments (count == num_trials, num_reqs == ops_per_trial) */ - for (i=0; i -#include - -#include "Trios_logger.h" - - - -enum IO_METHODS { - NETPERF_WRITE_RDMA_SYNC=0, - NETPERF_WRITE_RDMA_ASYNC, - NETPERF_READ_RDMA_SYNC, - NETPERF_READ_RDMA_ASYNC -}; - -enum DIST_SCHEME { - NETPERF_BLOCK_DISTRIBUTION, - NETPERF_ROUND_ROBIN_DISTRIBUTION -}; - -enum MPI_IO_METHODS { - NETPERF_MPI_SEND=0, - NETPERF_MPI_ISEND, - NETPERF_MPI_RECV, - NETPERF_MPI_IRECV, - NETPERF_MPI_PUT, - NETPERF_MPI_GET -}; - - -/** - * Options and arguments passed to the client driver. - */ -struct netperf_args { - bool client_flag; - bool server_flag; - int num_servers; // used for the server only - int num_threads; // used for the server only - bool block_distribution; // how to assign clients to servers - std::string server_url; - int transport; - std::string transport_name; - int len; - int io_method; - std::string url_file; - std::string io_method_name; - log_level debug_level; - std::string logfile; - int num_trials; - int num_reqs; - std::string result_file; - std::string result_file_mode; - int timeout; - int delay; - int num_retries; - bool validate_flag; -}; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - - -#else /* K&R C */ -#endif - - - - -#ifdef __cplusplus -} -#endif - - -#endif /* NETPERF_CLIENT_H_ */ diff --git a/packages/trios/tests/netperf/netperf_debug.cpp b/packages/trios/tests/netperf/netperf_debug.cpp deleted file mode 100644 index 4db528b7b030..000000000000 --- a/packages/trios/tests/netperf/netperf_debug.cpp +++ /dev/null @@ -1,55 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file netperf_debug.cpp - * - */ - -#include "Trios_logger.h" -#include "netperf_debug.h" - -/** - * Initialize the netperf_debug_level variable to LOG_UNDEFINED. This - * tells the logger to use the value set by logger_init, if called. - */ -log_level netperf_debug_level = LOG_UNDEFINED; diff --git a/packages/trios/tests/netperf/netperf_debug.h b/packages/trios/tests/netperf/netperf_debug.h deleted file mode 100644 index 0f095b68a93c..000000000000 --- a/packages/trios/tests/netperf/netperf_debug.h +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * netperf_client.h - * - * Created on: Aug 22, 2011 - * Author: raoldfi - */ - -#ifndef NETPERF_DEBUG_H_ -#define NETPERF_DEBUG_H_ - -#include "Trios_logger.h" - -extern log_level netperf_debug_level; - -#endif /* NETPERF_CLIENT_H_ */ diff --git a/packages/trios/tests/netperf/netperf_mpi_client.cpp b/packages/trios/tests/netperf/netperf_mpi_client.cpp deleted file mode 100644 index 391451e40b44..000000000000 --- a/packages/trios/tests/netperf/netperf_mpi_client.cpp +++ /dev/null @@ -1,567 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file netperf-client.cpp - * - */ - -/** - * @defgroup netperf_client Data Transfer Client - * - * @ingroup netperf_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "netperf_client.h" -#include "netperf_debug.h" -#include "netperf_util.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Teuchos_CommandLineProcessor.hpp" - -#include "netperf_service_args.h" - - - -log_level client_debug_level = LOG_UNDEFINED; - - -extern int print_args( - std::ostream &out, - const struct netperf_args &args, - const char *prefix); - - - -/** - * @brief Transfer an array of data structures using the MPI send/receive functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int netperf_mpi_send( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - data_array_t *array) -{ - int rc = 0; - MPI_Status status; - log_level debug_level = netperf_debug_level; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - int req_buf[3]; - req_buf[0] = len; - req_buf[1] = seed; - req_buf[2] = (int)validate; - - int nbytes = len * sizeof(struct data_t); - - log_debug(debug_level, "%d: Sending send req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - - // The request is just an MPI send with the number of elements - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, NETPERF_MPI_SEND_REQ_TAG, MPI_COMM_WORLD); - - log_debug(debug_level, "%d: Waiting for ack from %d to before sending data", rank, server_rank); - // Receive an ack that the server is ready - MPI_Recv(&rc, 0, MPI_INT, server_rank, NETPERF_MPI_SEND_ACK_TAG, MPI_COMM_WORLD, &status); - - - log_debug(debug_level, "%d: Send data: %d structs, %d bytes to %d", rank, len, nbytes, server_rank); - - // Now we send the data - rc = MPI_Send(&array->data_array_t_val[0], nbytes, - MPI_BYTE, server_rank, NETPERF_MPI_SEND_DATA_TAG, MPI_COMM_WORLD); - - return rc; -} - - -/** - * @brief Transfer an array of data structures using the MPI send/receive functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int netperf_mpi_isend( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array, - MPI_Request *req) -{ - int rc = 0; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Status status; - log_level debug_level = netperf_debug_level; - - int req_buf[3]; - req_buf[0] = (int)len; - req_buf[1] = (int)seed; - req_buf[2] = (int)validate; - - int nbytes = len * sizeof(data_t); - - log_debug(debug_level, "%d: Sending isend req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, NETPERF_MPI_ISEND_REQ_TAG, MPI_COMM_WORLD); - - // Receive an ack that the server is ready - MPI_Recv(&rc, 0, MPI_INT, server_rank, NETPERF_MPI_ISEND_ACK_TAG, MPI_COMM_WORLD, &status); - - log_debug(debug_level, "%d: Isend data: %d structs, %d bytes", rank, len, nbytes); - rc = MPI_Isend(array->data_array_t_val, nbytes, MPI_BYTE, server_rank, NETPERF_MPI_ISEND_DATA_TAG, MPI_COMM_WORLD, req); - - return rc; -} - - -/** - * @brief Transfer an array of data structures using the MPI put functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int netperf_mpi_put( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = 0; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Win win; - - int req_buf[3]; - req_buf[0] = (int)len; - req_buf[1] = (int)seed; - req_buf[2] = (int)validate; - int nbytes = len * sizeof(data_t); - - log_level debug_level = netperf_debug_level; - - - log_debug(debug_level, "%d: Sending put req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, NETPERF_MPI_PUT_REQ_TAG, MPI_COMM_WORLD); - - // The window on this side is not used to receive data - MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); - - MPI_Win_fence(0, win); - - - log_debug(debug_level, "%d: Put data: %d structs, %d bytes, val[0]=%d", rank, len, nbytes, - array->data_array_t_val[0].int_val); - - // Put the data from this window into a buffer on the server - MPI_Put(array->data_array_t_val, nbytes, MPI_BYTE, server_rank, 0, nbytes, MPI_BYTE, win); - - // This makes sure the operations are complete - MPI_Win_fence(0, win); - - MPI_Win_free(&win); - - return rc; -} - - -/** - * @brief Transfer an array of data structures using the MPI put functions. - * - * - * @param len The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int netperf_mpi_get( - const int server_rank, - const uint32_t len, - const uint32_t seed, - const bool validate, - const data_array_t *array) -{ - int rc = 0; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Win win; - - int req_buf[3]; - req_buf[0] = (int)len; - req_buf[1] = (int)seed; - req_buf[2] = (int)validate; - int nbytes = len * sizeof(data_t); - - log_level debug_level = netperf_debug_level; - - - log_debug(debug_level, "%d: Sending put req (len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - rc = MPI_Send(req_buf, 3, MPI_INT, server_rank, NETPERF_MPI_GET_REQ_TAG, MPI_COMM_WORLD); - - // The window on this side is not used to receive data - MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); - - MPI_Win_fence(0, win); - - - log_debug(debug_level, "%d: Get data: %d structs, %d bytes, val[0]=%d", rank, len, nbytes, - array->data_array_t_val[0].int_val); - - // Put the data from this window into a buffer on the server - MPI_Get(array->data_array_t_val, nbytes, MPI_BYTE, server_rank, 0, nbytes, MPI_BYTE, win); - - // This makes sure the operations are complete - MPI_Win_fence(0, win); - - MPI_Win_free(&win); - - return rc; -} - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param netperf_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -netperf_mpi_client_main (struct netperf_args &args, const int server_rank, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int i,j; - double time; - double start_time; - std::ofstream result_stream; - log_level debug_level = netperf_debug_level; - int client_rank, client_size; - int global_rank, global_size; - - /* the array of results (for async experiments) */ - std::vector results; - - netperf_read_encode_res *res4=NULL; - - std::vector timings; - std::vector timings_desc; - - /* unique to each process */ - int num_reqs; - - MPI_Comm_rank(MPI_COMM_WORLD, &global_rank); - MPI_Comm_size(MPI_COMM_WORLD, &global_size); - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - log_debug(debug_level, "%d: Starting client", global_rank); - - /* initialize logger */ -// if (args.logfile.empty()) { -// logger_init(args.debug_level, NULL); -// debug_level = args.debug_level; -// } else { -// char fn[1024]; -// sprintf(fn, "%s.%03d.log", args.logfile.c_str(), client_rank); -// logger_init(args.debug_level, fn); -// debug_level = args.debug_level; -// } - - - /* divide the requests among the processors */ - num_reqs = args.num_reqs; - /* the array of requests (for async experiments) */ - std::vector < MPI_Request > reqs(num_reqs); - std::vector < MPI_Status > status(num_reqs); - - /* open the result file */ - if (client_rank == 0) { - - if (!args.result_file.empty()) { - - if (args.result_file_mode.compare("a") == 0) - result_stream.open(args.result_file.c_str(), fstream::out | fstream::app); - else - result_stream.open(args.result_file.c_str(), fstream::out); - - if (!result_stream.is_open()) { - log_warn(client_debug_level, - "invalid result file:" - "defaults to stdout"); - } - } - } - - - /* allocate space for the data arrays used for testing */ - std::vector array_vec; - array_vec.reserve(args.num_reqs); // async jobs need num_reqs arrays - - for (i=0; i -#include - -#include -#include -#include -#include -#include -#include - - - -#include - -log_level netperf_debug_level = LOG_UNDEFINED; - - -#ifdef GNI_PERF -#include -extern gemini_state_t gni_state; -#endif - -/** - * @brief Send the data through the MPI_Send function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_send( - const int len, - const int seed, - const int validate, - const int source) -{ - int nbytes = len*sizeof(data_t); - MPI_Request req; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_level debug_level = netperf_debug_level; - int rc = 0; - MPI_Status status; - - log_debug(debug_level, "%d: starting process_mpi_send(len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - - log_debug(debug_level, "%d: Server Posting IRECV for %d structs, %d bytes, tag=%d", - rank, len, nbytes, NETPERF_MPI_SEND_DATA_TAG); - - // Post a receive for the buffer of ''len'' elements -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); -#endif - MPI_Irecv(array.data_array_t_val, nbytes, MPI_BYTE, source, NETPERF_MPI_SEND_DATA_TAG, MPI_COMM_WORLD, &req); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_send - MPI_Irecv data buf"); -#endif - - log_debug(debug_level, "%d: Server Sending \"READY ACK\" to %d", rank, source); - - // Send an ack to the client that the server is ready - rc = MPI_Send(&rc, 0, MPI_INT, source, NETPERF_MPI_SEND_ACK_TAG, MPI_COMM_WORLD); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_send - MPI_send rtr"); -#endif - - log_debug(debug_level, "%d: Server is waiting for data from %d, rc=%d", rank, source, rc); - - // Wait for the data to arrive - rc = MPI_Wait(&req, &status); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_send - MPI_wait data buf recv"); -#endif - log_debug(debug_level, "%d: MPI_Wait complete: rc=%d, status.MPI_ERROR=%d, status.MPI_SOURCE=%d", - rank, rc, status.MPI_ERROR, status.MPI_SOURCE); - - if (validate) { - rc = netperf_validate_array(seed, &array); - if (rc != 0) - log_warn(debug_level, "Invalid array"); - } - - // free the memory - delete [] array.data_array_t_val; - - return rc; -} - - -/** - * @brief Send the data through the MPI_Send function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_isend( - const int len, - const int seed, - const int validate, - const int source) -{ - int nbytes = len*sizeof(data_t); - - log_level debug_level = netperf_debug_level; - int rc=0; - MPI_Status status; - MPI_Request req; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(debug_level, "%d: starting process_mpi_isend(len=%d, seed=%d, validate=%d)", rank, len, seed, validate); - - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - log_debug(debug_level, "Posting IRECV"); - - // Post a receive for the buffer of ''len'' elements -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); -#endif - MPI_Irecv(array.data_array_t_val, nbytes, MPI_BYTE, source, NETPERF_MPI_ISEND_DATA_TAG, MPI_COMM_WORLD, &req); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_isend - MPI_Irecv data buf"); -#endif - - log_debug(debug_level, "Sending \"READY ACK\""); - - // Send an ack to the client that the server is ready - MPI_Send(&rc, 0, MPI_INT, source, NETPERF_MPI_ISEND_ACK_TAG, MPI_COMM_WORLD); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_isend - MPI_send rtr"); -#endif - - log_debug(debug_level, "Waiting for data"); - - // Wait for the data to arrive - MPI_Wait(&req, &status); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_isend - MPI_wait data buf recv"); -#endif - - if (validate) { - rc = netperf_validate_array(seed, &array); - if (rc != 0) - log_warn(debug_level, "Invalid array"); - } - - // Do we need to send a result? - delete [] array.data_array_t_val; - - return rc; -} - - -/** - * @brief Send the data through the MPI_Put function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_put( - const int len, - const int seed, - const int validate, - const int source) -{ - log_level debug_level = netperf_debug_level; - int rc = 0; - MPI_Win win; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(debug_level, "%d: starting process_mpi_put(len=%d, seed=%d, validate=%d)", - rank, len, seed, validate); - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - log_debug(debug_level, "%d: creating window", rank); - - - // Collective call to create a window for the data being transferred (like registering the memory) -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); -#endif - MPI_Win_create(array.data_array_t_val, sizeof(data_t)*len, - 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_put - MPI_Win_create data win"); -#endif - - MPI_Win_fence(0, win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_put - MPI_Win_fence1 data win"); -#endif - log_debug(debug_level, "Waiting for data"); - MPI_Win_fence(0, win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_put - MPI_Win_fence2 data win"); -#endif - - log_debug(debug_level, "Data should be here... validate if necessary, val[0]=%d",array.data_array_t_val[0].int_val); - - if (validate) { - rc = netperf_validate_array(seed, &array); - if (rc == 0) { - log_info(debug_level, "Validate Passed"); - } - else { - log_warn(debug_level, "Invalid array"); - } - - log_info(debug_level, "after validate, rc=%d", rc); - } - - // clean up - MPI_Win_free(&win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_put - MPI_Win_free data win"); -#endif - - delete [] array.data_array_t_val; - - return rc; -} - - -/** - * @brief Client reads the data using the MPI_Get function - * - * @param len Number of data structures sent. - * @param source MPI rank of the sending process. - */ -int process_mpi_get( - const int len, - const int seed, - const int validate, - const int source) -{ - log_level debug_level = netperf_debug_level; - int rc = 0; - MPI_Win win; - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - log_debug(debug_level, "%d: starting process_mpi_get(len=%d, seed=%d, validate=%d)", - rank, len, seed, validate); - - // Allocate space for the incoming buffer - data_array_t array; - array.data_array_t_len = len; - array.data_array_t_val = new data_t[len]; - - log_debug(debug_level, "%d: creating window", rank); - - // initialize the array - if (validate) { - netperf_init_data_array(seed, &array); - } - - // Collective call to create a window for the data being transferred (like registering the memory) -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); -#endif - MPI_Win_create(array.data_array_t_val, sizeof(data_t)*len, - 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_get - MPI_Win_create data win"); -#endif - - MPI_Win_fence(0, win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_get - MPI_Win_fence1 data win"); -#endif - log_debug(debug_level, "Waiting for client to get data"); - MPI_Win_fence(0, win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_get - MPI_Win_fence2 data win"); -#endif - - log_debug(debug_level, "Data should be transferred"); - - // clean up - MPI_Win_free(&win); -#ifdef GNI_PERF - gemini_read_counters(MPI_COMM_WORLD, &gni_state); - gemini_print_counters(MPI_COMM_WORLD, &gni_state, "process_mpi_get - MPI_Win_free data win"); -#endif - - delete [] array.data_array_t_val; - - return rc; -} - - -void make_progress(bool is_idle) -{ - log_debug(netperf_debug_level, "current_time(%llu) is_idle(%llu)", (uint64_t)trios_get_time_ms(), (uint64_t)is_idle); - - return; -} - - -/** - * @brief The NSSI netperf-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int netperf_mpi_server_main(MPI_Comm server_comm) -{ - int rc = 0; - - log_level debug_level = netperf_debug_level; - int server_rank; - - MPI_Comm_rank(server_comm, &server_rank); - - log_debug(debug_level, "%d: Starting server", server_rank); - - /* options that can be overriden by the command-line */ - std::string logfile(""); - - bool done=false; - MPI_Status status; - -#ifdef GNI_PERF - gemini_init_state(server_comm, &gni_state); -#endif - - // Server loop just waits for requests from any of the clients. The - // server isn't expecting any particular type of request. - - while (!done) { - int req_buf[3]; - - log_debug(debug_level, "Waiting for request"); - - // Receive the next request, the tag identifies the type of request - MPI_Recv(&req_buf, 3, MPI_INT, MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &status); - int len = req_buf[0]; - int seed = req_buf[1]; - int validate = req_buf[2]; - - log_debug(debug_level, "Server Received Request (source=%d, tag=%d, len=%d, seed=%d, validate=%d)", - status.MPI_SOURCE, status.MPI_TAG, len, seed, validate); - - switch (status.MPI_TAG) { - - case NETPERF_MPI_SEND_REQ_TAG: - rc = process_mpi_send(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing SEND request"); - break; - - case NETPERF_MPI_ISEND_REQ_TAG: - log_debug(debug_level, "Server Calling ISEND"); - rc = process_mpi_isend(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing ISEND request"); - break; - - case NETPERF_MPI_PUT_REQ_TAG: - log_debug(debug_level, "Server Calling PUT"); - rc = process_mpi_put(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing PUT request"); - break; - - case NETPERF_MPI_GET_REQ_TAG: - log_debug(debug_level, "Server Calling PUT"); - rc = process_mpi_get(len, seed, validate, status.MPI_SOURCE); - if (rc != 0) - log_error(debug_level, "Error processing GET request"); - break; - - case NETPERF_MPI_FINI_REQ: - log_debug(debug_level, "Server Received EXIT"); - done = true; - break; - - default: - log_error(debug_level, "tag=%d is not supported", status.MPI_TAG); - return -1; - } - } - - log_debug(debug_level, "Exiting server"); - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/netperf/netperf_mpi_test.cpp b/packages/trios/tests/netperf/netperf_mpi_test.cpp deleted file mode 100644 index c1f9256d6763..000000000000 --- a/packages/trios/tests/netperf/netperf_mpi_test.cpp +++ /dev/null @@ -1,372 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - - -/* - * netperf_service_test.cpp - * - * Created on: Aug 22, 2011 - * Author: raoldfi - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include "netperf_client.h" -#include "netperf_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -int netperf_mpi_server_main(MPI_Comm server_comm); -int netperf_mpi_client_main (struct netperf_args &args, const int server_rank, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct netperf_args &args, - const char *prefix) -{ - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url << std::endl; - - if (args.client_flag) { - out << prefix << " \tio-method = " << args.io_method_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tlen = " << args.len << std::endl; - out << prefix << " \tvalidate = " << ((args.validate_flag)?"true":"false") << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct netperf_args args; - - const int num_io_methods = 6; - const int io_method_vals[] = { - NETPERF_MPI_SEND, NETPERF_MPI_ISEND, - NETPERF_MPI_RECV, NETPERF_MPI_IRECV, - NETPERF_MPI_PUT, NETPERF_MPI_GET}; - const char * io_method_names[] = { - "send", "isend", - "recv", "irecv", - "put", "get"}; - - - // Initialize arguments - args.len = 1; - args.delay = 1; - args.io_method = NETPERF_MPI_SEND; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.timeout = 500; - args.num_retries = 5; - args.validate_flag = true; - args.server_url = ""; - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("len", &args.len, "The number of structures in an input buffer"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("validate", "no-validate", &args.validate_flag, "Validate the data"); - - // Set an enumeration command line option for the io_method - - parser.setOption("io-method", &args.io_method, num_io_methods, io_method_vals, io_method_names, - "I/O Methods for the example: \n" - "\t\t\tsend : Write data using the MPI_Send function\n" - "\t\t\tisend: Write data using the MPI_Isend function \n" - "\t\t\trecv : Read data using MPI_Recv function\n" - "\t\t\tirecv : Read data using the MPI_Irecv function\n" - "\t\t\tput: Write data using the MPI_Put function\n" - "\t\t\tget: Read data through the MPI_Get function"); - - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - debug_level = netperf_debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting netperf-mpi test", rank); - - - /** - * We need to play some fancy MPI games to get the communicators working correctly. - * We're running as both a client and a server in the same app, so we need to - * split the communicator so that the client can barrier sync without involving the server. - */ - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - int server_rank = 0; - int color = (rank == server_rank) ? 0 : 1; // only one server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - -// std::cout << "rank=" << rank << "/" << np << ", color=" << color << -// ", new_rank=" << splitrank << "/" << splitsize << std::endl << std::endl; - - - - // Set the debug level for the netperf service. - netperf_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - args.io_method_name = io_method_names[args.io_method]; - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (rank == server_rank) { - rc = netperf_mpi_server_main(comm); - log_debug(debug_level, "Server is finished, rc=%d", rc); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. */ - else { - log_debug(debug_level, "Starting client main"); - // Start the client code - rc = netperf_mpi_client_main(args, server_rank, comm); - log_debug(debug_level, "Client is finished, rc=%d", rc); - } - - MPI_Barrier(MPI_COMM_WORLD); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - - if (!success) { - log_warn(debug_level, "%d: success=FALSE", rank); - } - if (rc != 0) { - log_warn(debug_level, "%d: rc=%d", rank, rc); - } - - if(success && (rc == 0)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/netperf/netperf_server.cpp b/packages/trios/tests/netperf/netperf_server.cpp deleted file mode 100644 index fc61a3f998c9..000000000000 --- a/packages/trios/tests/netperf/netperf_server.cpp +++ /dev/null @@ -1,413 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file netperf-server.cpp - * - * @brief Example data transfer server. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - */ - -/** - * @defgroup netperf_server Data Transfer Server - * - * @ingroup netperf_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -//#include "nssi_types.h" -#include "Trios_nssi_server.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nssi_debug.h" - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "netperf_util.h" -#include "netperf_threads.h" - - - -#include - -log_level netperf_debug_level = LOG_UNDEFINED; - - -#ifdef GNI_PERF -#include -extern gemini_state_t gni_state; -#endif - -struct netperf_experiment_data { - uint32_t num_reqs; - int32_t array_len; - bool validate; - data_array_t *transfer_array; - data_array_t *validation_array; -}; -typedef struct netperf_experiment_data netperf_experiment_data_t; - -static netperf_experiment_data_t experiment_data; - -/** - * @brief - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data. - * @param res_addr The remote memory descriptor for the result. - */ -int netperf_experiment_setup_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const netperf_experiment_setup_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NSSI_OK; - log_level debug_level = netperf_debug_level; - - experiment_data.num_reqs = args->num_reqs; - experiment_data.array_len = args->array_len; - experiment_data.validate = args->validate; - - int nbytes = experiment_data.array_len*sizeof(data_t); - - log_debug(debug_level, "starting netperf_experiment_setup_srvr"); - - /* allocate and initialize space for the transfer buffers */ - experiment_data.transfer_array=(data_array_t *)malloc(experiment_data.num_reqs*sizeof(data_array_t)); - for (uint32_t i=0; ibuffer_index; - int nbytes = experiment_data.array_len*sizeof(data_t); - - double get_time=0.0; - - log_debug(debug_level, "starting netperf_write_rdma_srvr"); - - log_debug(debug_level, "getting data from client (%s)", caller->url); - - /* now we need to fetch the data from the client */ - get_time = Trios::GetTime(); - rc = nssi_get_data(caller, experiment_data.transfer_array[buffer_index].data_array_t_val, nbytes, data_addr); - get_time = Trios::GetTime() - get_time; - if (rc != NSSI_OK) { - log_warn(debug_level, "could not fetch data from client"); - return rc; - } - - log_debug(LOG_ALL, "nssi_get_data() bandwidth = %e", (double)nbytes/1024/1024/get_time); - - /* Validate the array */ - if (experiment_data.validate && (rc == 0)) { - rc = netperf_compare_data_arrays(&experiment_data.validation_array[buffer_index], &experiment_data.transfer_array[buffer_index]); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - } - - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - - return rc; -} - - -/** - * @brief Emulate a read operation where the bulk data is sent using - * the \ref nssi_put function. - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int netperf_read_rdma_srvr( - const unsigned long request_id, - const NNTI_peer_t *caller, - const netperf_read_rdma_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc; - log_level debug_level = netperf_debug_level; - - const uint32_t buffer_index = args->buffer_index; - int nbytes = experiment_data.array_len*sizeof(data_t); - - double put_time=0.0; - - /* process array (nothing to do) */ - log_debug(debug_level, "starting netperf_read_rdma_srvr"); - - log_debug(debug_level, "putting data in client buffer"); - - /* now we need to put the data to the client */ - put_time = Trios::GetTime(); - rc = nssi_put_data(caller, experiment_data.transfer_array[buffer_index].data_array_t_val, nbytes, data_addr, -1); - put_time = Trios::GetTime() - put_time; - if (rc != NSSI_OK) { - log_warn(debug_level, "could not put data to client"); - return rc; - } - - log_debug(LOG_ALL, "nssi_put_data() bandwidth = %e", (double)nbytes/1024/1024/put_time); - - rc = nssi_send_result(caller, request_id, NSSI_OK, NULL, res_addr); - - return rc; -} - - - -void make_progress(bool is_idle) -{ - log_debug(netperf_debug_level, "current_time(%llu) is_idle(%llu)", (uint64_t)trios_get_time_ms(), (uint64_t)is_idle); - - return; -} - - -/** - * @brief The NSSI netperf-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int netperf_server_main(nssi_rpc_transport transport, int num_threads, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service netperf_svc; - int server_rank; - - MPI_Comm_rank(server_comm, &server_rank); - - /* options that can be overriden by the command-line */ - std::string server_url(NSSI_URL_LEN, '\0'); /* NNTI-style url of the server */ - std::string logfile(""); - - - memset(&netperf_svc, 0, sizeof(nssi_service)); - - -// log_str=logfile.c_str(); -// if (logfile.c_str()[0]=='\0') { -// log_str=NULL; -// } -// /* initialize and enable logging */ -// logger_init((log_level)verbose, NULL); -// debug_level = (log_level)verbose; - - - /* initialize the nssi service */ - rc = nssi_service_init(transport, NSSI_SHORT_REQUEST_SIZE, &netperf_svc); - if (rc != NSSI_OK) { - log_error(netperf_debug_level, "could not init netperf_svc: %s", - nssi_err_str(rc)); - return -1; - } - - // register callbacks for the service methods - NSSI_REGISTER_SERVER_STUB(NETPERF_EXPERIMENT_SETUP_OP, netperf_experiment_setup_srvr, netperf_experiment_setup_args, void); - NSSI_REGISTER_SERVER_STUB(NETPERF_EXPERIMENT_TEARDOWN_OP, netperf_experiment_teardown_srvr, void, void); - NSSI_REGISTER_SERVER_STUB(NETPERF_WRITE_RDMA_OP, netperf_write_rdma_srvr, netperf_write_rdma_args, void); - NSSI_REGISTER_SERVER_STUB(NETPERF_READ_RDMA_OP, netperf_read_rdma_srvr, netperf_read_rdma_args, void); - -#ifdef GNI_PERF - gemini_init_state(server_comm, &gni_state); -#endif - - // Get the Server URL - std::string url(NSSI_URL_LEN, '\0'); - nssi_get_url(transport, &url[0], NSSI_URL_LEN); - - - // Set the maxumum number of requests to handle (-1 == infinite) - netperf_svc.max_reqs = -1; - // netperf_svc.progress_callback=(uint64_t)make_progress; - // netperf_svc.progress_callback_timeout=100; - - log_debug(netperf_debug_level, "Starting Server: url = %s", url.c_str()); - - // Tell the NSSI server to output log data - //rpc_debug_level = netperf_debug_level; - - // Start processing requests, the client will send a request to exit when done. - // If we're running multithreaded, we need to replace the process_request function with the - // enqueue_reqs function and start the process_queue_reqs thread. - - if (num_threads > 0) { - log_debug(netperf_debug_level, "Starting server threads"); - - netperf_start_server_threads(num_threads, 1000); - - // The main thread will execute unit it receives a "kill" request - rc = nssi_service_start_wfn(&netperf_svc, &netperf_enqueue_rpc_request); - - netperf_cancel_server_threads(); - } - else { - rc = nssi_service_start(&netperf_svc); - if (rc != NSSI_OK) { - log_info(netperf_debug_level, "exited netperf_svc: %s", - nssi_err_str(rc)); - } - } - - - - sleep(5); - - /* shutdown the netperf_svc */ - log_debug(netperf_debug_level, "shutting down service library"); - nssi_service_fini(&netperf_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/netperf/netperf_service_args.x b/packages/trios/tests/netperf/netperf_service_args.x deleted file mode 100644 index 996c4ba2e1ae..000000000000 --- a/packages/trios/tests/netperf/netperf_service_args.x +++ /dev/null @@ -1,203 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* -------------------------------------------------------------------------- */ -/** - * @file netperf_xdr.x - * - * @brief Type definitions for a simple rpc service. - * - * @author Ron Oldfield (raoldfi\@cs.sandia.gov). - * $Revision: 342 $. - * $Date: 2005-05-01 23:30:57 -0600 (Sun, 01 May 2005) $. - * - */ - -/** - * @defgroup netperf_example Nessie Data Transfer Example - * - * The data-transfer example demonstrates a simple client - * and server that transfer an array of 16-byte \ref data_t - * data structures from a parallel application to a set of - * servers. We implemented two variations: - * one that transfers the array of data structures - * with the request, and a second method that has each server - * pull the data using the \ref nssi_get_data() function. Although - * this example is fairly simple, it makes a decent benchmark code - * to evaluate overheads of the Nessie transfer protocols and encoding - * schemes. - * -*/ - -/** - * @defgroup netperf_types Nessie Example Types - * @ingroup netperf_example - * - * @{ - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file. */ -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -/** - * @brief Opcodes for the types of transfer operations. - */ -enum netperf_op { - /** Opcode for client to tell server experiment parameters. */ - NETPERF_EXPERIMENT_SETUP_OP=1, - /** Opcode for client to tell server to conclude experiment. */ - NETPERF_EXPERIMENT_TEARDOWN_OP, - /** Opcode for the writing the data through the data channel. */ - NETPERF_WRITE_RDMA_OP, - /** Opcode for reading the data throught the data channel. */ - NETPERF_READ_RDMA_OP -}; - -/** - * @brief Opcodes for the MPI transfer tests - */ -enum netperf_mpi_tag { - /** Opcode for blocking send request */ - NETPERF_MPI_SEND_REQ_TAG = 1001, - NETPERF_MPI_SEND_ACK_TAG, - NETPERF_MPI_SEND_DATA_TAG, - /** Opcode for non-blocking send request */ - NETPERF_MPI_ISEND_REQ_TAG, - NETPERF_MPI_ISEND_ACK_TAG, - NETPERF_MPI_ISEND_DATA_TAG, - /** Opcode for blocking recv request */ - NETPERF_MPI_RECV_REQ_TAG, - NETPERF_MPI_RECV_ACK_TAG, - NETPERF_MPI_RECV_DATA_TAG, - /** Opcode for non-blocking recv request */ - NETPERF_MPI_IRECV_REQ_TAG, - NETPERF_MPI_IRECV_ACK_TAG, - NETPERF_MPI_IRECV_DATA_TAG, - /** Opcode for one-sided put request */ - NETPERF_MPI_PUT_REQ_TAG, - NETPERF_MPI_PUT_ACK_TAG, - NETPERF_MPI_PUT_DATA_TAG, - /** Opcode for one-sided get request */ - NETPERF_MPI_GET_REQ_TAG, - NETPERF_MPI_GET_ACK_TAG, - NETPERF_MPI_GET_DATA_TAG, - /** Opcode to quit the server */ - NETPERF_MPI_FINI_REQ -}; - -/** - * @brief A 16-byte structure that contains an int, float, and double. - * - * This structure contains an int, float, and double as an example - * of a complex structure with multiple types. This will exercise the - * encoding/decoding features of Nessie. - */ -struct data_t { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -}; - -/** - * @brief Array of 16-byte structures that we can send with the request. - * - * Rpcgen will use this definition to define encoding functions to - * encode and decode an array of \ref data_t structures. We will - * use these functions when sending the array with the request. - */ -typedef data_t data_array_t<>; - -/** - * @brief Arguments for NETPERF_EXPERIMENT_SETUP. - * - * Communicate experiment parameters to server. - */ -struct netperf_experiment_setup_args { - /** The number of requests per trial. The server will create and - * initialize this many data arrays. */ - uint32_t num_reqs; - /** The length of each data array. */ - int32_t array_len; - /** A flag to perform validation. Determines if server will - * create validation arrays. */ - bool validate; -}; - -/** - * @brief Arguments for the first transfer operation, NETPERF_WRITE_RDMA. - * - * The first transfer operation only needs to send the index of the - * data array to transfer. It uses the data argument in the nssi_call_rpc() - * to identify the raw data for the server to get. - */ -struct netperf_write_rdma_args { - /** Index the array */ - uint32_t buffer_index; -}; - -/** - * @brief Arguments for the second transfer operation, NETPERF_READ_RDMA. - * - * The second transfer operation only needs to send the index of the - * data array to transfer. It uses the data argument in the nssi_call_rpc() - * to identify the raw data for the server to put. - */ -struct netperf_read_rdma_args { - /** Index the array */ - uint32_t buffer_index; -}; - -/** - * @} - */ diff --git a/packages/trios/tests/netperf/netperf_service_test.cpp b/packages/trios/tests/netperf/netperf_service_test.cpp deleted file mode 100644 index 7a492e72dac8..000000000000 --- a/packages/trios/tests/netperf/netperf_service_test.cpp +++ /dev/null @@ -1,633 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -/* - * netperf_service_test.cpp - * - * Created on: Aug 22, 2011 - * Author: raoldfi - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" -#include "Trios_nssi_debug.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include "netperf_client.h" -#include "netperf_debug.h" -#include "netperf_util.h" - - -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -// Prototypes for client and server codes -int netperf_server_main(nssi_rpc_transport transport, int num_threads, MPI_Comm server_comm); -int netperf_client_main (struct netperf_args &args, nssi_service &netperf_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - - -int print_args( - std::ostream &out, - const struct netperf_args &args, - const char *prefix) -{ - - if (args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (client and server) ----------- " << std::endl; - else if (args.client_flag && !args.server_flag) - out << prefix << " ------------ ARGUMENTS (client) ----------- " << std::endl; - else if (!args.client_flag && args.server_flag) - out << prefix << " ------------ ARGUMENTS (server) ----------- " << std::endl; - - out << prefix << " \tserver-url = " << args.server_url.c_str() << std::endl; - - if (args.client_flag) { - out << prefix << " \ttransport = " << args.transport_name << std::endl; - out << prefix << " \tio-method = " << args.io_method_name << std::endl; - out << prefix << " \tnum-trials = " << args.num_trials << std::endl; - out << prefix << " \tnum-reqs = " << args.num_reqs << std::endl; - out << prefix << " \tlen = " << args.len << std::endl; - out << prefix << " \tvalidate = " << ((args.validate_flag)?"true":"false") << std::endl; - out << prefix << " \tresult-file = " << - (args.result_file.empty()?"":args.result_file) << std::endl; - out << prefix << " \tresult-file-mode = " << args.result_file_mode << std::endl; - } - out << prefix << " \tdebug = " << args.debug_level << std::endl; - out << prefix << " \tlogfile = " << args.logfile << std::endl; - out << prefix << " ----------------------------------- " << std::endl; - - return 0; -} - - - - - -int main(int argc, char *argv[]) -{ - int np=1, rank=0; - int splitrank, splitsize; - int rc = 0; - nssi_service netperf_svc; - - int server_index=0; - int rank_in_server=0; - - int transport_index=-1; - - MPI_Init(&argc, &argv); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &np); - - MPI_Barrier(MPI_COMM_WORLD); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 0 ? std::cout : blackhole ); - - struct netperf_args args; - - const int num_io_methods = 4; - const int io_method_vals[] = { - NETPERF_WRITE_RDMA_SYNC, NETPERF_WRITE_RDMA_ASYNC, - NETPERF_READ_RDMA_SYNC, NETPERF_READ_RDMA_ASYNC}; - const char * io_method_names[] = { - "write-rdma-sync", "write-rdma-async", - "read-rdma-sync", "read-rdma-async"}; - - const int nssi_transport_list[] = { - NSSI_RPC_PTL, - NSSI_RPC_PTL, - NSSI_RPC_IB, - NSSI_RPC_IB, - NSSI_RPC_GEMINI, - NSSI_RPC_GEMINI, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGPDCMF, - NSSI_RPC_BGQPAMI, - NSSI_RPC_BGQPAMI, - NSSI_RPC_MPI}; - - const int num_nssi_transports = 11; - const int nssi_transport_vals[] = { - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10 - }; - const char * nssi_transport_names[] = { - "portals", - "ptl", - "infiniband", - "ib", - "gemini", - "gni", - "bgpdcmf", - "dcmf", - "bgqpami", - "pami", - "mpi" - }; - - - // Initialize arguments - args.transport=NSSI_DEFAULT_TRANSPORT; - args.len = 1; - args.delay = 1; - args.io_method = NETPERF_WRITE_RDMA_SYNC; - args.debug_level = LOG_WARN; - args.num_trials = 1; - args.num_reqs = 1; - args.result_file_mode = "a"; - args.result_file = ""; - args.url_file = ""; - args.logfile = ""; - args.client_flag = true; - args.server_flag = true; - args.num_servers = 1; - args.num_threads = 0; - args.timeout = 500; - args.num_retries = 5; - args.validate_flag = true; - args.block_distribution = true; - - - bool success = true; - - /** - * We make extensive use of the \ref Teuchos::CommandLineProcessor for command-line - * options to control the behavior of the test code. To evaluate performance, - * the "num-trials", "num-reqs", and "len" options control the amount of data transferred - * between client and server. The "io-method" selects the type of data transfer. The - * server-url specifies the URL of the server. If running as a server, the server-url - * provides a recommended URL when initializing the network transport. - */ - try { - - //out << Teuchos::Teuchos_Version() << std::endl << std::endl; - - // Creating an empty command line processor looks like: - Teuchos::CommandLineProcessor parser; - parser.setDocString( - "This example program demonstrates a simple data-transfer service " - "built using the NEtwork Scalable Service Interface (Nessie)." - ); - - /* To set and option, it must be given a name and default value. Additionally, - each option can be given a help std::string. Although it is not necessary, a help - std::string aids a users comprehension of the acceptable command line arguments. - Some examples of setting command line options are: - */ - - parser.setOption("delay", &args.delay, "time(s) for client to wait for server to start" ); - parser.setOption("timeout", &args.timeout, "time(ms) to wait for server to respond" ); - parser.setOption("server", "no-server", &args.server_flag, "Run the server" ); - parser.setOption("client", "no-client", &args.client_flag, "Run the client"); - parser.setOption("len", &args.len, "The number of structures in an input buffer"); - parser.setOption("debug",(int*)(&args.debug_level), "Debug level"); - parser.setOption("logfile", &args.logfile, "log file"); - parser.setOption("num-trials", &args.num_trials, "Number of trials (experiments)"); - parser.setOption("num-reqs", &args.num_reqs, "Number of reqs/trial"); - parser.setOption("result-file", &args.result_file, "Where to store results"); - parser.setOption("result-file-mode", &args.result_file_mode, "Write mode for the result"); - parser.setOption("server-url-file", &args.url_file, "File that has URL client uses to find server"); - parser.setOption("validate", "no-validate", &args.validate_flag, "Validate the data"); - parser.setOption("num-servers", &args.num_servers, "Number of server processes"); - parser.setOption("num-threads", &args.num_threads, "Number of threads used by each server process"); - parser.setOption("block-distribution", "rr-distribution", &args.block_distribution, - "Use a block distribution scheme to assign clients to servers"); - - // Set an enumeration command line option for the io_method - parser.setOption("io-method", &args.io_method, num_io_methods, io_method_vals, io_method_names, - "I/O Methods for the example: \n" - "\t\t\twrite-rdma-sync : Write data using RDMA (server pulls) - synchronous\n" - "\t\t\twrite-rdma-async: Write data using RDMA (server pulls) - asynchronous\n" - "\t\t\tread-rdma-sync : Read data using RDMA (server puts) - synchronous\n" - "\t\t\tread-rdma-async: Read data using RDMA (server puts) - asynchronous"); - - - // Set an enumeration command line option for the NNTI transport - parser.setOption("transport", &transport_index, num_nssi_transports, nssi_transport_vals, nssi_transport_names, - "NSSI transports (not all are available on every platform): \n" - "\t\t\tportals|ptl : Cray or Schutt\n" - "\t\t\tinfiniband|ib : libibverbs\n" - "\t\t\tgemini|gni : Cray libugni (Gemini or Aries)\n" - "\t\t\tbgpdcmf|dcmf : IBM BG/P DCMF\n" - "\t\t\tbgqpami|pami : IBM BG/Q PAMI\n" - "\t\t\tmpi : isend/irecv implementation\n" - ); - - - - /* There are also two methods that control the behavior of the - command line processor. First, for the command line processor to - allow an unrecognized a command line option to be ignored (and - only have a warning printed), use: - */ - parser.recogniseAllOptions(true); - - /* Second, by default, if the parser finds a command line option it - doesn't recognize or finds the --help option, it will throw an - std::exception. If you want prevent a command line processor from - throwing an std::exception (which is important in this program since - we don't have an try/catch around this) when it encounters a - unrecognized option or help is printed, use: - */ - parser.throwExceptions(false); - - /* We now parse the command line where argc and argv are passed to - the parse method. Note that since we have turned off std::exception - throwing above we had better grab the return argument so that - we can see what happened and act accordingly. - */ - Teuchos::CommandLineProcessor::EParseCommandLineReturn parseReturn= parser.parse( argc, argv ); - - if( parseReturn == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED ) { - return 0; - } - - if( parseReturn != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) { - return 1; // Error! - - } - - // Here is where you would use these command line arguments but for this example program - // we will just print the help message with the new values of the command-line arguments. - //if (rank == 0) - // out << "\nPrinting help message with new values of command-line arguments ...\n\n"; - - //parser.printHelpMessage(argv[0],out); - - } - - TEUCHOS_STANDARD_CATCH_STATEMENTS(true,std::cerr,success); - - log_debug(LOG_ALL, "transport_index=%d", transport_index); - if (transport_index > -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - args.io_method_name=std::string(io_method_names[args.io_method]); - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting netperf-service test", rank); - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - int color = 0; // color=0-->server, color=1-->client - if (args.client_flag && args.server_flag) { - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Put all the servers as the first ranks. - if (rank < args.num_servers) { - color = 0; - log_debug(debug_level, "rank=%d is a server", rank); - } - else { - color = 1; // all others are clients - log_debug(debug_level, "rank=%d is a client", rank); - } - - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - } - else { - if (args.client_flag) - color=1; - else if (args.server_flag) - color=0; - else { - log_error(debug_level, "Must be either a client or a server"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - } - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - log_debug(debug_level, "%d: Finished splitting communicators", rank); - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // If running as both client and server, gather and distribute - // the server URLs to all the clients. - if (args.server_flag && args.client_flag) { - - std::string all_urls; - - // This needs to be a vector of chars, not a string - all_urls.resize(args.num_servers * NSSI_URL_LEN, '\0'); - - // Have servers gather their URLs - if (color == 0) { - assert(args.num_servers == splitsize); // these should be equal - - log_debug(debug_level, "%d: Gathering urls: my_url=%s", rank, my_url.c_str()); - - // gather all urls to rank 0 of the server comm (also rank 0 of MPI_COMM_WORLD) - MPI_Gather(&my_url[0], NSSI_URL_LEN, MPI_CHAR, - &all_urls[0], NSSI_URL_LEN, MPI_CHAR, 0, comm); - } - - // broadcast the full set of server urls to all processes - MPI_Bcast(&all_urls[0], all_urls.size(), MPI_CHAR, 0, MPI_COMM_WORLD); - - log_debug(debug_level, "%d: Bcast urls, urls.size=%d", rank, all_urls.size()); - - if (color == 1) { - - // For block distribution scheme use the utility function (in netperf_util.cpp) - if (args.block_distribution) { - // Use this utility function to calculate the server_index - netperf_block_partition(args.num_servers, splitsize, splitrank, &server_index, &rank_in_server); - } - - // Use a simple round robin distribution scheme - else { - server_index = splitrank % args.num_servers; - rank_in_server = splitrank / args.num_servers; - } - - // Copy the server url out of the list of urls - int offset = server_index * NSSI_URL_LEN; - - args.server_url = all_urls.substr(offset, NSSI_URL_LEN); - - log_debug(debug_level, "client %d assigned to server \"%s\"", splitrank, args.server_url.c_str()); - } - - - log_debug(debug_level, "%d: Finished distributing server urls, server_url=%s", rank, args.server_url.c_str()); - } - - // If running as a client only, have to get the list of servers from the urlfile. - else if (!args.server_flag && args.client_flag){ - - sleep(args.delay); // give server time to get started - - std::vector< std::string > urlbuf; - netperf_read_server_url_file(args.url_file.c_str(), urlbuf, comm); - args.num_servers = urlbuf.size(); - - // For block distribution scheme use the utility function (in netperf_util.cpp) - if (args.block_distribution) { - // Use this utility function to calculate the server_index - netperf_block_partition(args.num_servers, splitsize, splitrank, &server_index, &rank_in_server); - } - - // Use a simple round robin distribution scheme - else { - server_index = splitrank % args.num_servers; - rank_in_server = splitrank / args.num_servers; - } - - args.server_url = urlbuf[server_index]; - log_debug(debug_level, "client %d assigned to server \"%s\"", splitrank, args.server_url.c_str()); - } - - else if (args.server_flag && !args.client_flag) { - args.server_url = my_url; - - if (args.url_file.empty()) { - log_error(debug_level, "Must set --url-file"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - netperf_write_server_url_file(args.url_file.c_str(), my_url.c_str(), comm); - } - - // Set the debug level for the netperf service. - netperf_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - log_debug(debug_level, "%d: server_url=%s", rank, args.server_url.c_str()); - - print_args(out, args, "%"); - - log_debug(debug_level, "server_url=%s", args.server_url.c_str()); - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (color == 0) { - rc = netperf_server_main((nssi_rpc_transport)args.transport, args.num_threads, comm); - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, node 0, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref netperf_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d, url=%s", i, args.server_url.c_str()); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url.c_str(), args.timeout, &netperf_svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(netperf_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - // wait for all the clients to connect - MPI_Barrier(comm); - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(netperf_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&netperf_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - netperf_client_main(args, netperf_svc, comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if (rank_in_server == 0) { - log_debug(debug_level, "%d: Halting netperf service", rank); - rc = nssi_kill(&netperf_svc, 0, 5000); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/netperf/netperf_threads.cpp b/packages/trios/tests/netperf/netperf_threads.cpp deleted file mode 100644 index 67c538b89c01..000000000000 --- a/packages/trios/tests/netperf/netperf_threads.cpp +++ /dev/null @@ -1,180 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * netperf_threads.cpp - * - * Created on: Aug 20, 2012 - * Author: raoldfi - */ - -#include "Trios_nssi_server.h" -#include "Trios_logger.h" - -#include -#include - -#include "netperf_debug.h" -#include "netperf_threads.h" - - - -std::queue pending_reqs; -pthread_cond_t pending_cond; -pthread_mutex_t pending_mutex; - -static volatile bool time_to_exit=false; - -static uint32_t max_num_reqs; -static std::queue consumer_threads; - -/** This is the consumer thread code. - * It pops requests off the pending queue and calls the - * process_rpc_request function. - */ -void *process_pending_reqs(void *arg) -{ - int rc; - nssi_svc_rpc_request *req = NULL; - log_level debug_level = netperf_debug_level; - - intptr_t id = (intptr_t)arg; - - log_info(debug_level, "%d: starting thread", id); - - // do this outside the loop to prevent tons of calls - pthread_mutex_lock(&pending_mutex); - - while (!time_to_exit) { - - if (!pending_reqs.empty()) { - // pull the next request off the queue - req = pending_reqs.front(); - pending_reqs.pop(); - - // signal threads that the state of the queue has changed - pthread_cond_signal(&pending_cond); - - // unlock the mutex while we process the request - pthread_mutex_unlock(&pending_mutex); - - log_debug(debug_level, "%d: processing request %d", id, req->id); - - rc = nssi_process_rpc_request(req); - if (rc) { - log_error(netperf_debug_level, "%d: Error processing request", id); - return NULL; - } - pthread_mutex_lock(&pending_mutex); - } - else { - // this will block this thread until someone sends us a signal - pthread_cond_wait(&pending_cond, &pending_mutex); - } - } - - pthread_mutex_unlock(&pending_mutex); - - log_info(debug_level, "%d: exiting process_pending_reqs thread", id); - return NULL; -} - - - -void netperf_start_server_threads(const int num_threads, const int max_reqs) -{ - // initialize the condition and mutex variables for the pending queue - pthread_cond_init(&pending_cond, NULL); // default attributes - pthread_mutex_init(&pending_mutex, NULL); // default attributes - - max_num_reqs = max_reqs; - - // start the consumer threads - for (int64_t i=0; i= max_num_reqs) { - pthread_cond_wait(&pending_cond, &pending_mutex); - } - - log_debug(netperf_debug_level, "Adding request %d to the pending queue", req->id); - - // ok to add the request - pending_reqs.push(req); - - // tell the processing threads the queue has changed - pthread_cond_signal(&pending_cond); - - pthread_mutex_unlock(&pending_mutex); - - return(0); -} - -void netperf_cancel_server_threads() -{ - log_debug(netperf_debug_level, "Canceling server threads"); - - pthread_mutex_lock(&pending_mutex); - time_to_exit = true; - pthread_cond_broadcast(&pending_cond); - pthread_mutex_unlock(&pending_mutex); - - while (!consumer_threads.empty()) { - pthread_t tid = consumer_threads.front(); - consumer_threads.pop(); - pthread_join(tid, NULL); - } -} - diff --git a/packages/trios/tests/netperf/netperf_threads.h b/packages/trios/tests/netperf/netperf_threads.h deleted file mode 100644 index 4285cb573918..000000000000 --- a/packages/trios/tests/netperf/netperf_threads.h +++ /dev/null @@ -1,60 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * netperf_threads.h - * - * Created on: Aug 20, 2012 - * Author: raoldfi - */ - -#ifndef NETPERF_THREADS_H_ -#define NETPERF_THREADS_H_ - -#include "Trios_nssi_server.h" - -void netperf_start_server_threads(const int num_threads, const int max_reqs); -int netperf_enqueue_rpc_request(nssi_svc_rpc_request *req); -void netperf_cancel_server_threads(); - - -#endif /* NETPERF_THREADS_H_ */ diff --git a/packages/trios/tests/netperf/netperf_util.cpp b/packages/trios/tests/netperf/netperf_util.cpp deleted file mode 100644 index 35a16c225c1c..000000000000 --- a/packages/trios/tests/netperf/netperf_util.cpp +++ /dev/null @@ -1,310 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * netperf_util.cpp - * - * Created on: Nov 4, 2011 - * Author: raoldfi - */ - - -#include "Trios_nssi_types_xdr.h" -#include "netperf_service_args.h" -#include "netperf_debug.h" -#include "netperf_util.h" -#include "netperf_client.h" - -#include -#include -#include -#include -#include -#include - - - -/** - * Output the configuration file with this format - * ------ - * line 1: ns # number of servers - * line 2: server_url_0 - * line 3: server_url_1 - * ... - * line n: server_url_n-1 - */ -int netperf_write_server_url_file( - std::string url_fname, - std::string my_url, - MPI_Comm comm) -{ - int rc = 0; - int rank, np; - log_level debug_level = LOG_UNDEFINED; - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &np); - - // output the config file - if (!url_fname.empty()) { - - // gather the urls from all the servers - std::string urls(NSSI_URL_LEN*np, '\0'); - MPI_Gather(&my_url[0], NSSI_URL_LEN, MPI_CHAR, &urls[0], NSSI_URL_LEN, MPI_CHAR, 0, comm); - - // Write the urls to the config file - if (rank == 0) { - int i; - - std::ofstream urlfile (url_fname.c_str()); - if (urlfile.is_open()) { - // write the number of servers as the first line - urlfile << np << std::endl; - - for (i=0; i &urlbuf, MPI_Comm comm) -{ - log_level debug_level = netperf_debug_level; - int rank, np; - - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &np); - - // variables for the serverURLs - int numServers = 0; - - // Open the file and get all the server urls - if (rank == 0) { - - // try to open the file - std::ifstream urlFile(path); - if (urlFile.is_open()) { - if (urlFile.good()) { - std::string line; - std::getline(urlFile, line); - numServers = std::atoi(line.c_str()); - } - - urlbuf.resize(numServers, std::string(NSSI_URL_LEN, '\0')); - - for (int i=0; idata_array_t_len; - data_t *buf = array->data_array_t_val; - - /* initialize the random seed */ - //initstate(seed, state, 8); - - for (int i=0; idata_array_t_len != arr2->data_array_t_len) { - log_error(debug_level, "arr1->len=%d, arr2->len=%d", - arr1->data_array_t_len, arr2->data_array_t_len); - return -1; - } - - for (int i=0; i<(int)arr1->data_array_t_len; i++) { - if (arr1->data_array_t_val[i].int_val != arr2->data_array_t_val[i].int_val) { - log_error(debug_level, "val[%d].int_val=%d, val[%d].int_val=%d", - i,arr1->data_array_t_val[i].int_val, - i,arr2->data_array_t_val[i].int_val); - return -1; - } - if (arr1->data_array_t_val[i].float_val != arr2->data_array_t_val[i].float_val) { - log_error(debug_level, "val[%d].float_val=%f, val[%d].float_val=%f", - i,arr1->data_array_t_val[i].float_val, - i,arr2->data_array_t_val[i].float_val); - return -1; - } - if (arr1->data_array_t_val[i].double_val != arr2->data_array_t_val[i].double_val) { - log_error(debug_level, "val[%d].double_val=%g, val[%d].double_val=%g", - i,arr1->data_array_t_val[i].double_val, - i,arr2->data_array_t_val[i].double_val); - return -1; - } - } - - return 0; - -} - - -int netperf_validate_array(const int seed, data_array_t *array) -{ - int rc = 0; - log_level debug_level = netperf_debug_level; - - /* Validate the array that was sent through the args */ - data_array_t tmp_array; - - tmp_array.data_array_t_len = array->data_array_t_len; - tmp_array.data_array_t_val = new data_t[array->data_array_t_len]; - - netperf_init_data_array(seed, &tmp_array); - rc = netperf_compare_data_arrays(&tmp_array, array); - - if (rc != 0) { - log_warn(debug_level, "Unable to validate array"); - } - - delete [] tmp_array.data_array_t_val; - return rc; -} - - -/** - * Partition this val into a bin using a simple linear block-partitioning - * scheme. - * - * Returns the bin assignment and the rank within that bin. - * - */ -int netperf_block_partition( - const int num_bins, - const int num_vals, - const int val, - int *bin, - int *rank) -{ - - int per_bin = num_vals / num_bins; - int num_large = num_vals % num_bins; // number of bins with one extra - - int barrier = num_large * (per_bin + 1); // total number of vals in large bins - - if (val < barrier) { - *bin = val / (per_bin + 1); - *rank = val % (per_bin + 1); - } - - else { - *bin = num_large + (val - barrier)/per_bin; - *rank = (val - barrier) % per_bin; - } - - //std::cout << "num_bins=" << num_bins << ", num_vals=" << num_vals; - //std::cout << ": val=" << val << ", bin=" << bin << ", rank=" << rank << std::endl; - - return 0; -} diff --git a/packages/trios/tests/netperf/netperf_util.h b/packages/trios/tests/netperf/netperf_util.h deleted file mode 100644 index 1104e3d9e2f3..000000000000 --- a/packages/trios/tests/netperf/netperf_util.h +++ /dev/null @@ -1,112 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * netperf_util.h - * - * Created on: May 31, 2012 - * Author: raoldfi - */ - -#ifndef NETPERF_UTIL_H_ -#define NETPERF_UTIL_H_ - -#include -#include -#include - -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - - -#include "netperf_service_args.h" - -#ifdef __cplusplus -extern "C" { -#endif - -extern int netperf_write_server_url_file( - std::string url_fname, - std::string my_url, - MPI_Comm comm); - - -extern void netperf_read_server_url_file( - const char *path, - std::vector &urlbuf, - MPI_Comm comm); - -/** - * Initialize an array given a starting seed. - * - * This function is used by both the client and the server to generate - * initialize values in a buffer. - */ -extern void netperf_init_data_array( - const unsigned int seed, - data_array_t *array); - -/** - * Compare two arrays. If they are equal, return 0. - */ -extern int netperf_compare_data_arrays( - const data_array_t *arr1, - const data_array_t *arr2); - - -extern int netperf_validate_array( - const int seed, - data_array_t *array); - -extern int netperf_block_partition( - const int num_bins, - const int num_vals, - const int val, - int *bin, - int *rank); - - -#ifdef __cplusplus -} -#endif - -#endif /* NETPERF_UTIL_H_ */ diff --git a/packages/trios/tests/nssi_send/CMakeLists.txt b/packages/trios/tests/nssi_send/CMakeLists.txt deleted file mode 100644 index 1f44523c7610..000000000000 --- a/packages/trios/tests/nssi_send/CMakeLists.txt +++ /dev/null @@ -1,83 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -############# Special exception for rpcgen extra commas in enum ########## -IF (CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") -ENDIF (CMAKE_COMPILER_IS_GNUCXX) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -# We want the binary directory to be a system directory because it contains -# code generated by rpcgen, may have many warnings. -#INCLUDE_DIRECTORIES(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}) - -# Generate the XDR header and source file for the service arguments -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/nssi_send_service_args.x) - - - -TRIBITS_ADD_EXECUTABLE( - NssiSendTest - SOURCES nssi_send_service_args.c nssi_send_test.cpp nssi_send_server.cpp nssi_send_client.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -TRIBITS_ADD_TEST( - NssiSendTest - NOEXEPREFIX - NAME NssiSendTest - COMM serial mpi - NUM_MPI_PROCS 2) diff --git a/packages/trios/tests/nssi_send/cmake/Dependencies.cmake b/packages/trios/tests/nssi_send/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/tests/nssi_send/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/nssi_send/nssi_send_client.cpp b/packages/trios/tests/nssi_send/nssi_send_client.cpp deleted file mode 100644 index 6662a06d5cc0..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_client.cpp +++ /dev/null @@ -1,728 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * send_client.cpp - * - * Created on: Sep 25, 2014 - * Author: thkorde - */ - -/** - * @defgroup send_client Data Transfer Client - * - * @ingroup send_test - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nnti.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "Teuchos_CommandLineProcessor.hpp" - -#include "nssi_send_test.h" -#include "nssi_send_service_args.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - - - -log_level client_debug_level = LOG_UNDEFINED; - -extern int print_args( - std::ostream &out, - const struct send_args &args, - const char *prefix); -uint64_t calc_checksum (const char * buf, const uint64_t size); - - -#define MAX_COUNT 100 - - -#define REQUEST_FLAG_APP_PINNED_SHORT_REQUEST (1 << 0) -#define REQUEST_FLAG_APP_PINNED_LONG_ARGS (1 << 1) -#define REQUEST_FLAG_APP_PINNED_BULK_DATA (1 << 2) -#define REQUEST_FLAG_APP_PINNED_SHORT_RESULT (1 << 3) - -#define REQUEST_FLAG_SYNC (1 << 4) -#define REQUEST_FLAG_RESPONSELESS (1 << 5) - -#define REQUEST_APP_PINNED_FLAG_COUNT 4 - - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. The server sends the data back via RDMA. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. The server is expected to send the data - * back using nssi_put_data(). - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int send_no_rdma( - const nssi_service *svc, - uint32_t flags, - NNTI_buffer_t *short_request_hdl, - NNTI_buffer_t *long_args_hdl, - NNTI_buffer_t *bulk_data_hdl, - NNTI_buffer_t *short_result_hdl, - int32_t count, - send_test_args *args, - void *data, - uint32_t data_size, - send_test_res *res, - nssi_request *req) -{ - int rc = NSSI_OK; - log_level debug_level = client_debug_level; - - log_debug(debug_level, "Calling RPC for SEND_TEST_NO_RDMA_OP"); - - rc = nssi_create_request( - svc, - SEND_TEST_NO_RDMA_OP, - args, - data, - data_size, - res, - req); - - if (flags & REQUEST_FLAG_APP_PINNED_SHORT_REQUEST) { - if (short_request_hdl == NULL) { - MPI_Abort(MPI_COMM_WORLD, -5); - } - req->app_pinned_short_request=TRUE; - req->short_request_hdl = short_request_hdl; - } - if (flags & REQUEST_FLAG_APP_PINNED_LONG_ARGS) { - if (long_args_hdl == NULL) { - MPI_Abort(MPI_COMM_WORLD, -5); - } - req->app_pinned_long_args=TRUE; - req->long_args_hdl = long_args_hdl; - } - if (flags & REQUEST_FLAG_APP_PINNED_BULK_DATA) { - if (bulk_data_hdl == NULL) { - MPI_Abort(MPI_COMM_WORLD, -5); - } - req->app_pinned_bulk_data=TRUE; - req->bulk_data_hdl = bulk_data_hdl; - } - if (flags & REQUEST_FLAG_APP_PINNED_SHORT_RESULT) { - if (short_result_hdl == NULL) { - MPI_Abort(MPI_COMM_WORLD, -5); - } - req->app_pinned_short_result=TRUE; - req->short_result_hdl = short_result_hdl; - } - - if (flags & REQUEST_FLAG_SYNC) { - req->is_sync=TRUE; - } - if (flags & REQUEST_FLAG_RESPONSELESS) { - req->is_responseless=TRUE; - req->result =NULL; - req->short_result_hdl=NULL; - } - - /* initialize the arguments */ - memset(args, 0, sizeof(send_test_args)); - - args->count = count; - args->array.data_array_t_len = count; - args->array.data_array_t_val = (data_t *)calloc(count, sizeof(data_t)); - - for (int i=0;iarray.data_array_t_val->int_val = i + 10; - args->array.data_array_t_val->float_val = i + 10.10; - args->array.data_array_t_val->double_val = i + 10.10; - } - args->args_chksum = calc_checksum((const char*)args->array.data_array_t_val, sizeof(count*sizeof(data_t))); - - args->bulk_chksum = 0; - - /* call the remote methods */ - rc = nssi_send_request(req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call OPREG_EMPTY_REQUEST_OP: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array with the request structure. The server sends the data back in the result. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int send_no_rdma_blk( - const nssi_service *svc, - uint32_t flags, - NNTI_buffer_t *short_request_hdl, - NNTI_buffer_t *long_args_hdl, - NNTI_buffer_t *bulk_data_hdl, - NNTI_buffer_t *short_result_hdl, - int32_t count, - send_test_args *args, - void *data, - uint32_t data_size, - send_test_res *res) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - - nssi_request req; - - if ((flags & REQUEST_FLAG_SYNC) || (flags & REQUEST_FLAG_RESPONSELESS)) { - log_error(client_debug_level, "This is the nssi_send_request() + nssi_wait() test - cannot be combined with REQUEST_FLAG_SYNC or REQUEST_FLAG_RESPONSELESS"); - rc = NSSI_EINVAL; - goto out; - } - - /* call the async function */ - rc = send_no_rdma( - svc, - flags, - short_request_hdl, - long_args_hdl, - bulk_data_hdl, - short_result_hdl, - count, - args, - data, - data_size, - res, - &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - -out: - return rc; - -} - -int execute_send_no_rdma( - const nssi_service *svc, - uint32_t flags, - NNTI_buffer_t *short_request_hdl, - NNTI_buffer_t *long_args_hdl, - NNTI_buffer_t *bulk_data_hdl, - NNTI_buffer_t *short_result_hdl, - int32_t count, - void *data, - uint32_t data_size) -{ - int rc = NSSI_OK; - - send_test_args args; - send_test_res res; - - if ((flags & REQUEST_FLAG_SYNC) || (flags & REQUEST_FLAG_RESPONSELESS)) { - nssi_request req; - - rc = send_no_rdma( - svc, - flags, - short_request_hdl, - long_args_hdl, - bulk_data_hdl, - short_result_hdl, - count, - &args, - data, - data_size, - &res, - &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - } else { - rc = send_no_rdma_blk( - svc, - flags, - short_request_hdl, - long_args_hdl, - bulk_data_hdl, - short_result_hdl, - count, - &args, - data, - data_size, - &res); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - } - - if (!(flags & REQUEST_FLAG_RESPONSELESS)) { - if (res.args_chksum != args.args_chksum) { - log_error(client_debug_level, "server returned a different checksum: args.args_chksum(%llu) != res.args_chksum(%llu)", args.args_chksum, res.args_chksum); - rc=NSSI_EINVAL; - } - if (res.bulk_chksum != args.bulk_chksum) { - log_error(client_debug_level, "server returned a different checksum: args.bulk_chksum(%llu) != res.bulk_chksum(%llu)", args.bulk_chksum, res.bulk_chksum); - rc=NSSI_EINVAL; - } - } - - return rc; -} - - -int read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return(-1); - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return(1); - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(client_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); - - return(0); -} - - -extern NNTI_transport_t transports[NSSI_RPC_COUNT]; - -int setup_handles( - const nssi_service *svc, - uint32_t flags, - NNTI_buffer_t *short_request_hdl, - NNTI_buffer_t *long_args_hdl, - NNTI_buffer_t *bulk_data_hdl, - void *data, - uint32_t data_size, - NNTI_buffer_t *short_result_hdl) -{ - int rc=NNTI_OK; - - int short_request_len = NNTI_BUFFER_SIZE(&svc->req_addr); - int long_args_len = 5120; - - - if (flags & REQUEST_FLAG_APP_PINNED_SHORT_REQUEST) { - rc=NNTI_alloc( - &transports[svc->transport_id], - short_request_len, - 1, - NNTI_SEND_SRC, - short_request_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed registering short request: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - if (flags & REQUEST_FLAG_APP_PINNED_LONG_ARGS) { - rc=NNTI_alloc( - &transports[svc->transport_id], - long_args_len, - 1, - NNTI_GET_SRC, - long_args_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed registering long args: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - if (flags & REQUEST_FLAG_APP_PINNED_BULK_DATA) { - rc=NNTI_register_memory( - &transports[svc->transport_id], - (char *)data, - data_size, - 1, - (NNTI_buf_ops_t)(NNTI_GET_SRC|NNTI_PUT_DST), - bulk_data_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed registering data: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - if (flags & REQUEST_FLAG_APP_PINNED_SHORT_RESULT) { - rc=NNTI_alloc( - &transports[svc->transport_id], - NSSI_SHORT_RESULT_SIZE, - 1, - NNTI_RECV_DST, - short_result_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed registering short result: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - -cleanup: - return rc; -} - - -int teardown_handles( - uint32_t flags, - NNTI_buffer_t *short_request_hdl, - NNTI_buffer_t *long_args_hdl, - NNTI_buffer_t *bulk_data_hdl, - NNTI_buffer_t *short_result_hdl) -{ - int rc=NNTI_OK; - - if (flags & REQUEST_FLAG_APP_PINNED_SHORT_REQUEST) { - rc=NNTI_free( - short_request_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed freeing short request: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - if (flags & REQUEST_FLAG_APP_PINNED_LONG_ARGS) { - rc=NNTI_free( - long_args_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed freeing long args: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - if (flags & REQUEST_FLAG_APP_PINNED_BULK_DATA) { - rc=NNTI_unregister_memory( - bulk_data_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed unregistering data: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - if (flags & REQUEST_FLAG_APP_PINNED_SHORT_RESULT) { - rc=NNTI_free( - short_result_hdl); - if (rc != NNTI_OK) { - log_error(client_debug_level, "failed freeing short result: %s", - nnti_err_str(rc)); - goto cleanup; - } - } - -cleanup: - return rc; -} - - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param send_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -send_client_main (struct send_cmdline_args &args, nssi_service &send_svc, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int client_rank, client_size; - - uint32_t flags=0; - - NNTI_buffer_t short_request_hdl; - NNTI_buffer_t long_args_hdl; - NNTI_buffer_t bulk_data_hdl; - NNTI_buffer_t short_result_hdl; - - int32_t count=1; - - void *data=NULL; - uint32_t data_size=0; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(SEND_TEST_NO_RDMA_OP, send_test_args, void, send_test_res); - NSSI_REGISTER_CLIENT_STUB(SEND_TEST_SRV_GET_OP, send_test_args, void, send_test_res); - NSSI_REGISTER_CLIENT_STUB(SEND_TEST_SRV_PUT_OP, send_test_args, void, send_test_res); - - data_size=MAX_COUNT*sizeof(data_t); - data=malloc(data_size); - - - - count=1; - - - /* First send a traditional RPC request. - * NSSI is responsible for allocating and pinning all memory regions. - * nssi_wait() is explicitly called to get the result. - */ - flags=0; - rc = execute_send_no_rdma( - &send_svc, - flags, - &short_request_hdl, - &long_args_hdl, - &bulk_data_hdl, - &short_result_hdl, - count, - data, - data_size); - if (rc != NSSI_OK) { - log_error(client_debug_level, "could not transfer data: %s", - nssi_err_str(rc)); - goto abort; - } - - /* Next send a traditional RPC request but skip the nssi_wait(). - * NSSI is responsible for allocating and pinning all memory regions. - * Set req->is_sync which causes an implicit nssi_wait() at the end of nssi_send_request(). - */ - flags = REQUEST_FLAG_SYNC; - rc = execute_send_no_rdma( - &send_svc, - flags, - &short_request_hdl, - &long_args_hdl, - &bulk_data_hdl, - &short_result_hdl, - count, - data, - data_size); - if (rc != NSSI_OK) { - log_error(client_debug_level, "could not transfer data: %s", - nssi_err_str(rc)); - goto abort; - } - - /* Next send a traditional RPC request but skip the nssi_wait(). - * NSSI is responsible for allocating and pinning all memory regions. - * Set req->is_responseless which causes the server to skip sending the result. - */ - flags = REQUEST_FLAG_RESPONSELESS; - rc = execute_send_no_rdma( - &send_svc, - flags, - &short_request_hdl, - &long_args_hdl, - &bulk_data_hdl, - &short_result_hdl, - count, - NULL, - 0); - if (rc != NSSI_OK) { - log_error(client_debug_level, "could not transfer data: %s", - nssi_err_str(rc)); - goto abort; - } - - - for (int i=0;iis_sync which causes an implicit nssi_wait() at the end of nssi_send_request(). - */ - flags=(1 << i) | REQUEST_FLAG_SYNC; - rc = execute_send_no_rdma( - &send_svc, - flags, - &short_request_hdl, - &long_args_hdl, - &bulk_data_hdl, - &short_result_hdl, - count, - data, - data_size); - if (rc != NSSI_OK) { - log_error(client_debug_level, "could not transfer data: %s", - nssi_err_str(rc)); - goto abort; - } - - - if (((1 << i)) == REQUEST_FLAG_APP_PINNED_SHORT_REQUEST) { - /* Next send a traditional RPC request but skip the nssi_wait(). - * NSSI is responsible for allocating and pinning all memory regions. - * Set req->is_responseless which causes the server to skip sending the result. - */ - flags=(1 << i) | REQUEST_FLAG_RESPONSELESS; - rc = execute_send_no_rdma( - &send_svc, - flags, - &short_request_hdl, - &long_args_hdl, - &bulk_data_hdl, - &short_result_hdl, - count, - NULL, - 0); - if (rc != NSSI_OK) { - log_error(client_debug_level, "could not transfer data: %s", - nssi_err_str(rc)); - goto abort; - } - } - - flags=(1 << i); - rc=teardown_handles( - flags, - &short_request_hdl, - &long_args_hdl, - &bulk_data_hdl, - &short_result_hdl); - } - - - if (client_rank==0) - cout << "experiment complete" << endl; - - - return 0; - -abort: - exit(2); -} - -/** - * @} - */ diff --git a/packages/trios/tests/nssi_send/nssi_send_debug.cpp b/packages/trios/tests/nssi_send/nssi_send_debug.cpp deleted file mode 100644 index 13caa9f13542..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_debug.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * send_debug.cpp - * - * Created on: Sep 25, 2014 - * Author: thkorde - */ - -#include "Trios_logger.h" -#include "send_debug.h" - -/** - * Initialize the opreg_debug_level variable to LOG_UNDEFINED. This - * tells the logger to use the value set by logger_init, if called. - */ -log_level send_debug_level = LOG_UNDEFINED; diff --git a/packages/trios/tests/nssi_send/nssi_send_debug.h b/packages/trios/tests/nssi_send/nssi_send_debug.h deleted file mode 100644 index b48da50f92de..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_debug.h +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * send_debug.h - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -#ifndef SEND_DEBUG_H_ -#define SEND_DEBUG_H_ - -#include "Trios_logger.h" - -extern log_level send_debug_level; - -#endif /* SEND_CLIENT_H_ */ diff --git a/packages/trios/tests/nssi_send/nssi_send_server.cpp b/packages/trios/tests/nssi_send/nssi_send_server.cpp deleted file mode 100644 index 6d60c11cf9d3..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_server.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file send-server.cpp - * - * @brief Example data transfer server. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - */ - -/** - * @defgroup send_server Data Transfer Server - * - * @ingroup send_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_server.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nssi_debug.h" -#include "Trios_nssi_fprint_types.h" - -#include "nssi_send_service_args.h" - -#include -#include -#include -#include -#include -#include - - - -log_level send_debug_level = LOG_UNDEFINED; - -uint64_t calc_checksum (const char * buf, const uint64_t size); - - -/** - * @brief - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int send_test_no_rdma( - const unsigned long request_id, - const NNTI_peer_t *caller, - const send_test_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NNTI_OK; - log_level debug_level = send_debug_level; - send_test_res res; - - /* process array (nothing to do) */ - log_debug(debug_level, "enter"); - - memset(&res, 0, sizeof(send_test_res)); - - /* checksum the data */ - res.args_chksum = calc_checksum((const char*)args->array.data_array_t_val, sizeof(args->count*sizeof(data_t))); - if (res.args_chksum != args->args_chksum) { - log_error(debug_level, "client args checksum (%lu) != calculated args checksum (%lu)", - args->args_chksum, res.args_chksum); - rc=NNTI_EBADRPC; - } - - res.bulk_chksum = 0; - if (res.bulk_chksum != args->bulk_chksum) { - log_error(debug_level, "client bulk checksum (%lu) != calculated bulk checksum (%lu)", - args->bulk_chksum, res.bulk_chksum); - rc=NNTI_EBADRPC; - } - - rc = nssi_send_result(caller, request_id, rc, &res, res_addr); - - log_debug(debug_level, "exit"); - - return rc; -} - - - -/** - * @brief The NSSI send-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int send_server_main(nssi_rpc_transport transport, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service send_svc; - int server_rank; - - char url[NSSI_URL_LEN]; - - MPI_Comm_rank(server_comm, &server_rank); - - - memset(&send_svc, 0, sizeof(nssi_service)); - - - /* initialize the nssi service */ - rc = nssi_service_init(transport, NSSI_SHORT_REQUEST_SIZE, &send_svc); - if (rc != NSSI_OK) { - log_error(send_debug_level, "could not init send_svc: %s", - nssi_err_str(rc)); - return -1; - } - - /* register callbacks for the service methods */ - NSSI_REGISTER_SERVER_STUB(SEND_TEST_NO_RDMA_OP, send_test_no_rdma, send_test_args, send_test_res); -// NSSI_REGISTER_SERVER_STUB(SEND_TEST_SRV_GET_OP, send_test_srv_get, send_test_args, send_test_res); -// NSSI_REGISTER_SERVER_STUB(SEND_TEST_SRV_PUT_OP, send_test_srv_put, send_test_args, send_test_res); - - - /* Get the Server URL */ - nssi_get_url(transport, &url[0], NSSI_URL_LEN); - - - /* Set the maxumum number of requests to handle (-1 == infinite) */ - send_svc.max_reqs = -1; - - log_debug(send_debug_level, "Starting Server: url = %s", url); - - /* start processing requests, the client will send a request to exit when done */ - rc = nssi_service_start(&send_svc); - if (rc != NSSI_OK) { - log_info(send_debug_level, "exited send_svc: %s", - nssi_err_str(rc)); - } - - sleep(5); - - /* shutdown the send_svc */ - log_debug(send_debug_level, "shutting down service library"); - nssi_service_fini(&send_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/nssi_send/nssi_send_service_args.x b/packages/trios/tests/nssi_send/nssi_send_service_args.x deleted file mode 100644 index 0078885934f3..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_service_args.x +++ /dev/null @@ -1,145 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file send_service_args.x - * - * @brief Type definitions for the send test. - * - * @author Todd kordenbrock (thkorde\@sandia.gov). - * - * Created on: Sep 25, 2014 - * - */ - -/** - * @defgroup send_test Exercise NSSI Send Options - * - */ - -/** - * @defgroup send_test_types NSSI Send Types - * @ingroup send_test - * - * @{ - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file. */ -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -enum send_test_type { - SEND_TEST_NO_RDMA = 0, - SEND_TEST_SRV_GET, - SEND_TEST_SRV_PUT -}; - -/** - * @brief Opcodes for the types of transfer operations. - */ -enum send_test_op { - /** Opcode for a request with no server-side RDMA of bulk data. */ - SEND_TEST_NO_RDMA_OP = 1, - - /** Opcode for a request with a server-side GET of bulk data. */ - SEND_TEST_SRV_GET_OP = 2, - - /** Opcode for a request with a server-side PUT of bulk data. */ - SEND_TEST_SRV_PUT_OP = 3 -}; - -/** - * @brief A 16-byte structure that contains an int, float, and double. - * - * This structure contains an int, float, and double as an example - * of a complex structure with multiple types. This will exercise the - * encoding/decoding features of Nessie. - */ -struct data_t { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -}; - -/** - * @brief Array of 16-byte structures that we can send with the request. - * - * Rpcgen will use this definition to define encoding functions to - * encode and decode an array of \ref data_t structures. We will - * use these functions when sending the array with the request. - */ -typedef data_t data_array_t<>; - - - -struct send_test_args { - /** Number of elements in the data array */ - int32_t count; - /** The array of \ref data_t structures, including length. */ - data_array_t array; - /** 64-bit checksum of the data array sent in the args */ - uint64_t args_chksum; - /** 64-bit checksum of the data array transfered by RDMA */ - uint64_t bulk_chksum; -}; - -struct send_test_res { - /** 64-bit checksum of the data array sent in the args */ - uint64_t args_chksum; - /** 64-bit checksum of the data array transfered by RDMA */ - uint64_t bulk_chksum; -}; - -/** - * @} - */ diff --git a/packages/trios/tests/nssi_send/nssi_send_test.cpp b/packages/trios/tests/nssi_send/nssi_send_test.cpp deleted file mode 100644 index 4fb4bc64142e..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_test.cpp +++ /dev/null @@ -1,546 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * sesnd_test.cpp - * - * Created on: Sep 25, 2014 - * Author: thkorde - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include "nssi_send_test.h" -#include "nssi_send_service_args.h" -#include "nssi_send_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -uint64_t calc_checksum (const char * buf, const uint64_t size); -int send_server_main(nssi_rpc_transport transport, MPI_Comm server_comm); -int send_client_main (struct send_cmdline_args &args, nssi_service &send_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - -uint64_t calc_checksum (const char * buf, const uint64_t size) -{ - unsigned long hash = 5381; - const char* p = buf; - uint64_t i = 0; - - for (i=0; i -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting send test", rank); - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - if (args.client_flag && args.server_flag) { - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - - int color = (rank == 0) ? 0 : 1; // only one server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - // std::cout << "rank=" << rank << "/" << np << ", color=" << color << - // ", new_rank=" << newrank << "/" << newsize << std::endl << std::endl; - // - // std::cout << "my_url=" << my_url << ", server_url=" << args.server_url << std::endl; - } - else { - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - } - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - if (args.server_flag && !args.server_url.empty()) { - // use the server URL as suggested URL - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, args.server_url.c_str()); - } - else { - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - } - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // Broadcast the server URL to all the clients - args.server_url.resize(NSSI_URL_LEN, '\0'); - if (args.server_flag && args.client_flag) { - args.server_url = my_url; - MPI_Bcast(&args.server_url[0], args.server_url.size(), MPI_CHAR, 0, MPI_COMM_WORLD); - } - - else if (!args.server_flag && args.client_flag){ - if (args.server_url.empty()) { - - // check to see if we're supposed to get the URL from a file - if (!args.url_file.empty()) { - // Fetch the server URL from a file - sleep(1); - log_debug(debug_level, "Reading from file %s", args.url_file.c_str()); - std::ifstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - if (urlfile.good()) - getline(urlfile, args.server_url); - } - else { - log_error(debug_level, "Failed to open server_url_file=%s", args.url_file.c_str()); - exit(1); - } - urlfile.close(); - log_debug(debug_level, "URL = %s", args.server_url.c_str()); - } - else { - log_error(debug_level, "Need to set --server-url=[ADDR] or --server-url-file=[PATH]"); - } - } - } - - else if (args.server_flag && !args.client_flag) { - args.server_url = my_url; - - // If the url_file value is set, write the url to a file - if (!args.url_file.empty()) { - std::ofstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - urlfile << args.server_url.c_str() << std::endl; - } - urlfile.close(); - log_debug(debug_level, "Wrote url to file %s", args.url_file.c_str()); - } - } - - - - // Set the debug level for the send service. - send_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (args.server_flag && (rank == 0)) { - rc = send_server_main((nssi_rpc_transport)args.transport, comm); - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, node 0, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref send_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - sleep(args.delay); // give server time to get started - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url.c_str(), args.timeout, &send_svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(send_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(send_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&send_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - send_client_main(args, send_svc, comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if (client_rank == 0) { - log_debug(debug_level, "%d: Halting send service", rank); - rc = nssi_kill(&send_svc, 0, 5000); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/nssi_send/nssi_send_test.h b/packages/trios/tests/nssi_send/nssi_send_test.h deleted file mode 100644 index 240bd3dc22fc..000000000000 --- a/packages/trios/tests/nssi_send/nssi_send_test.h +++ /dev/null @@ -1,79 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * send_client.h - * - * Created on: Sep 25, 2014 - * Author: thkorde - */ - -#ifndef SEND_CLIENT_H_ -#define SEND_CLIENT_H_ - -#include -#include - -#include "Trios_logger.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - - -/** - * Options and arguments passed to the client driver. - */ -struct send_cmdline_args { - bool client_flag; - bool server_flag; - int transport; - std::string transport_name; - std::string server_url; - std::string url_file; - log_level debug_level; - std::string logfile; - int timeout; - int delay; - int num_retries; -}; - - -#endif /* SEND_CLIENT_H_ */ diff --git a/packages/trios/tests/opreg/CMakeLists.txt b/packages/trios/tests/opreg/CMakeLists.txt deleted file mode 100644 index eb6f854fdaac..000000000000 --- a/packages/trios/tests/opreg/CMakeLists.txt +++ /dev/null @@ -1,96 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - -INCLUDE(TriosProcessXDR) - -############# Special exception for rpcgen extra commas in enum ########## -IF (CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isystem ${CMAKE_CURRENT_BINARY_DIR}") -ENDIF (CMAKE_COMPILER_IS_GNUCXX) - -INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) - - -# We want the binary directory to be a system directory because it contains -# code generated by rpcgen, may have many warnings. -#INCLUDE_DIRECTORIES(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}) - -# Generate the XDR header and source file for the service arguments -TriosProcessXDR(${CMAKE_CURRENT_SOURCE_DIR}/opreg_service_args.x) - - - -TRIBITS_ADD_EXECUTABLE( - OpRegTest - SOURCES opreg_service_args.c opreg_test.cpp opreg_c_server.c opreg_cpp_server.cpp opreg_client.cpp - DEPLIBS ${DEPLIBS} - NOEXEPREFIX -) - -SET(EXTRA_TEST_ARGS "") - -# Gemini network transport takes a few seconds to initialize -IF (${PACKAGE_NAME}_ENABLE_Gemini) - APPEND_SET(EXTRA_TEST_ARGS "--delay=3") -ENDIF() - - -SET(TESTNAMES - c-stub - cpp-obj) - - -foreach (testname ${TESTNAMES}) - - SET(DEFAULT_ARGS "--opreg-type=${testname} ") - - TRIBITS_ADD_TEST( - OpRegTest - NOEXEPREFIX - NAME OpRegTest_${testname} - ARGS "${DEFAULT_ARGS} ${EXTRA_TEST_ARGS}" - COMM serial mpi - NUM_MPI_PROCS 2) -endforeach() diff --git a/packages/trios/tests/opreg/cmake/Dependencies.cmake b/packages/trios/tests/opreg/cmake/Dependencies.cmake deleted file mode 100644 index 32714b27996c..000000000000 --- a/packages/trios/tests/opreg/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi TeuchosCore) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS MPI) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS) diff --git a/packages/trios/tests/opreg/opreg_c_server.c b/packages/trios/tests/opreg/opreg_c_server.c deleted file mode 100644 index 11b3d9995d0f..000000000000 --- a/packages/trios/tests/opreg/opreg_c_server.c +++ /dev/null @@ -1,191 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file opreg-server.cpp - * - * @brief Example data transfer server. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - */ - -/** - * @defgroup opreg_server Data Transfer Server - * - * @ingroup opreg_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_server.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nssi_debug.h" -#include "Trios_nssi_fprint_types.h" - -#include "opreg_service_args.h" - -#include -#include -#include -#include -#include -#include - - - -log_level opreg_debug_level = LOG_UNDEFINED; - -uint64_t calc_checksum (const char * buf, const uint64_t size); - - -/** - * @brief Emulate a read operation where the bulk data is sent using - * the \ref nssi_put function. - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -int opreg_request_processor( - const unsigned long request_id, - const NNTI_peer_t *caller, - const opreg_args *args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) -{ - int rc = NNTI_OK; - log_level debug_level = opreg_debug_level; - - /* process array (nothing to do) */ - log_debug(debug_level, "starting opreg_request_processor()"); - - log_debug(debug_level, "args->data.int_val(%d) args->data.float_val(%f) args->data.double_val(%f) args->chksum(%lu)", - args->data.int_val, args->data.float_val, args->data.double_val, args->chksum); - - /* checksum the data */ - if (args->chksum != calc_checksum((const char*)&args->data, sizeof(args->data))) { - log_error(debug_level, "client checksum (%lu) != calculated checksum (%lu)", - args->chksum, calc_checksum((const char*)&args->data, sizeof(args->data))); - rc=NNTI_EBADRPC; - } - - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - - return rc; -} - - - -/** - * @brief The NSSI opreg-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int opreg_c_server_main(nssi_rpc_transport transport, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service opreg_svc; - int server_rank; - - char url[NSSI_URL_LEN]; - - MPI_Comm_rank(server_comm, &server_rank); - - - memset(&opreg_svc, 0, sizeof(nssi_service)); - - - /* initialize the nssi service */ - rc = nssi_service_init(transport, NSSI_SHORT_REQUEST_SIZE, &opreg_svc); - if (rc != NSSI_OK) { - log_error(opreg_debug_level, "could not init opreg_svc: %s", - nssi_err_str(rc)); - return -1; - } - - /* register callbacks for the service methods */ - NSSI_REGISTER_SERVER_STUB(OPREG_REQUEST_OP, opreg_request_processor, opreg_args, void); - - - /* Get the Server URL */ - nssi_get_url(transport, &url[0], NSSI_URL_LEN); - - - /* Set the maxumum number of requests to handle (-1 == infinite) */ - opreg_svc.max_reqs = -1; - - log_debug(opreg_debug_level, "Starting Server: url = %s", url); - - /* start processing requests, the client will send a request to exit when done */ - rc = nssi_service_start(&opreg_svc); - if (rc != NSSI_OK) { - log_info(opreg_debug_level, "exited opreg_svc: %s", - nssi_err_str(rc)); - } - - sleep(5); - - /* shutdown the opreg_svc */ - log_debug(opreg_debug_level, "shutting down service library"); - nssi_service_fini(&opreg_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/opreg/opreg_client.cpp b/packages/trios/tests/opreg/opreg_client.cpp deleted file mode 100644 index ca51bd441975..000000000000 --- a/packages/trios/tests/opreg/opreg_client.cpp +++ /dev/null @@ -1,260 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * opreg_client.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -/** - * @defgroup opreg_client Data Transfer Client - * - * @ingroup opreg_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include "Teuchos_CommandLineProcessor.hpp" - -#include "opreg_test.h" -#include "opreg_service_args.h" - - -#include -#include -#include -#include -#include - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - - - -log_level client_debug_level = LOG_UNDEFINED; - -extern int print_args( - std::ostream &out, - const struct opreg_args &args, - const char *prefix); - -extern "C" { -uint64_t calc_checksum (const char * buf, const uint64_t size); -} - -/** - * @brief Asynchronous transfer of \ref data_t array with the request structure. The server sends the data back via RDMA. - * - * This function marshals the \ref data_array_t array of 16-byte - * structures with the request and send it to the server using the - * nssi_call_rpc() function. The server is expected to send the data - * back using nssi_put_data(). - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - * @param req The \ref nssi_request returned to caller. - */ -int opreg_request( - const nssi_service *svc, - nssi_request *req) -{ - int rc = NSSI_OK; - log_level debug_level = client_debug_level; - opreg_args args; - - log_debug(debug_level, "Calling RPC for OPREG_REQUEST_OP"); - - /* initialize the arguments */ - memset(&args, 0, sizeof(opreg_args)); - args.data.int_val = 10; - args.data.float_val = 10.10; - args.data.double_val = 10.10; - args.chksum = calc_checksum((const char*)&args.data, sizeof(args.data)); - - log_debug(debug_level, "args.data.int_val(%d) args.data.float_val(%f) args.data.double_val(%f) args.chksum(%lu)", - args.data.int_val, args.data.float_val, args.data.double_val, args.chksum); - - /* call the remote methods */ - rc = nssi_call_rpc(svc, OPREG_REQUEST_OP, &args, NULL, 0, NULL, req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call OPREG_EMPTY_REQUEST_OP: %s", - nssi_err_str(rc)); - } - - return rc; -} - -/** - * @brief Synchronous transfer of \ref data_t array with the request structure. The server sends the data back in the result. - * - * This method attaches the entire request buffer to a request - * and sends it to the server. This forces the client to encode - * the buffer before sending, and it makes the size of the request - * large, depending on the size of the data array. - * - * @param svc The service description of the remote service. - * @param array The data array to transfer. - */ -int opreg_request_blk( - const nssi_service *svc) -{ - int rc = NSSI_OK; - int rc2 = NSSI_OK; - nssi_request req; - - /* call the async function */ - rc = opreg_request(svc, &req); - if (rc != NSSI_OK) { - log_error(client_debug_level, "unable to call async method: %s", - nssi_err_str(rc)); - return rc; - } - - /* wait for completion */ - rc2 = nssi_wait(&req, &rc); - if (rc2 != NSSI_OK) { - log_error(client_debug_level, "failed waiting for request: %s", - nssi_err_str(rc)); - return rc2; - } - - if (rc != NSSI_OK) { - log_error(client_debug_level, "remote method failed: %s", - nssi_err_str(rc)); - return rc; - } - - return rc; - -} - - - -int read_contact_info(const char *fname, char *url, int maxlen) -{ - const char *contact_filename=NULL; - FILE *cf=NULL; - - if ((fname==NULL) || (fname[0]=='\0')) { - contact_filename=getenv("NNTI_CONTACT_FILENAME"); - } else { - contact_filename=fname; - } - if (contact_filename==NULL) { - url[0]='\0'; - return(-1); - } - cf=fopen(contact_filename, "r"); - if (cf == NULL) { - url[0]='\0'; - return(1); - } - if (fgets(url, maxlen, cf) == NULL) { - log_error(client_debug_level, "failed to read URL from %s", fname); - } - fclose(cf); - - return(0); -} - - - - -/** - * @brief Main code for data transfer client. - * - * @param args The options for the experiment, set at the command-line - * @param opreg_svc The nssi_service descriptor for the remote service (already connected) - * @param comm The communicator for the client application - */ -int -opreg_client_main (struct opreg_cmdline_args &args, nssi_service &opreg_svc, MPI_Comm client_comm) -{ - using namespace std; - - int rc; - int client_rank, client_size; - - - MPI_Comm_rank(client_comm, &client_rank); - MPI_Comm_size(client_comm, &client_size); - - - /* register the XDR encoding functions */ - NSSI_REGISTER_CLIENT_STUB(OPREG_REQUEST_OP, opreg_args, void, void); - - rc = opreg_request_blk(&opreg_svc); - if (rc != NSSI_OK) { - log_error(client_debug_level, "could not transfer data: %s", - nssi_err_str(rc)); - goto abort; - } - - if (client_rank==0) - cout << "experiment complete" << endl; - - - return 0; - -abort: - exit(2); -} - -/** - * @} - */ diff --git a/packages/trios/tests/opreg/opreg_cpp_server.cpp b/packages/trios/tests/opreg/opreg_cpp_server.cpp deleted file mode 100644 index 118b32083f93..000000000000 --- a/packages/trios/tests/opreg/opreg_cpp_server.cpp +++ /dev/null @@ -1,212 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** @file opreg-server.cpp - * - * @brief Example data transfer server. - * - * @author Ron Oldfield (raoldfi\@sandia.gov). - */ - -/** - * @defgroup opreg_server Data Transfer Server - * - * @ingroup opreg_example - * - * @{ - */ - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_server.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nssi_debug.h" -#include "Trios_nssi_fprint_types.h" - -#include "opreg_service_args.h" - -#include -#include - -#include -#include -#include -#include -#include -#include - - - -extern log_level opreg_debug_level; - -extern "C" { -uint64_t calc_checksum (const char * buf, const uint64_t size); -} - - -/** - * @brief Emulate a read operation where the bulk data is sent using - * the \ref nssi_put function. - * - * Transfer an array of data structures to the client using the data - * channel. This procedure passes the length of the array in the arguments. - * The server then ``puts'' the unencoded data into the client memory using - * the \ref nssi_put_data function. This method evaluates the RDMA - * transfer performance for \ref nssi_put_data. - * - * @param request_id ID of the request. - * @param caller The process ID of the calling process. - * @param args Arguments passed with the request. - * @param data_addr The remote memory descriptor for the data (not used). - * @param res_addr The remote memory descriptor for the result. - */ -class OpRegRequestProcessor : public NssiRpc { - public: - int doRPC( - const int opcode, - const unsigned long request_id, - const NNTI_peer_t *caller, - const void *void_args, - const NNTI_buffer_t *data_addr, - const NNTI_buffer_t *res_addr) - { - int rc = NNTI_OK; - log_level debug_level = opreg_debug_level; - const opreg_args *args = static_cast(void_args); - - /* process array (nothing to do) */ - log_debug(debug_level, "starting OpRegRequestProcessor->dispatch(opcode=%d)", opcode); - - log_debug(debug_level, "args->data.int_val(%d) args->data.float_val(%f) args->data.double_val(%f) args->chksum(%lu)", - args->data.int_val, args->data.float_val, args->data.double_val, args->chksum); - - /* checksum the data */ - if (args->chksum != calc_checksum((const char*)&args->data, sizeof(args->data))) { - log_error(debug_level, "client checksum (%lu) != calculated checksum (%lu)", - args->chksum, calc_checksum((const char*)&args->data, sizeof(args->data))); - rc=NNTI_EBADRPC; - } - - rc = nssi_send_result(caller, request_id, rc, NULL, res_addr); - - return rc; - } - void registerRPC() - { - NSSI_REGISTER_SERVER_OBJ(OPREG_REQUEST_OP, this, opreg_args, void); - } -}; - - -/** - * @brief The NSSI opreg-server. - * - * NSSI has already been initialized and the client already knows the URL of the - * server. This function simply registers the server methods and starts the - * service loop. The client will send a request to kill the service upon completion. - * - */ -int opreg_cpp_server_main(nssi_rpc_transport transport, MPI_Comm server_comm) -{ - int rc = NSSI_OK; - - nssi_service opreg_svc; - int server_rank; - - OpRegRequestProcessor *opreg_srvr_obj = new OpRegRequestProcessor(); - - MPI_Comm_rank(server_comm, &server_rank); - - /* options that can be overriden by the command-line */ - std::string server_url(NSSI_URL_LEN, '\0'); /* NNTI-style url of the server */ - std::string logfile(""); - - - memset(&opreg_svc, 0, sizeof(nssi_service)); - - - /* initialize the nssi service */ - rc = nssi_service_init(transport, NSSI_SHORT_REQUEST_SIZE, &opreg_svc); - if (rc != NSSI_OK) { - log_error(opreg_debug_level, "could not init opreg_svc: %s", - nssi_err_str(rc)); - return -1; - } - - // register objects for the service methods - opreg_srvr_obj->registerRPC(); - - - // Get the Server URL - std::string url(NSSI_URL_LEN, '\0'); - nssi_get_url(transport, &url[0], NSSI_URL_LEN); - - - // Set the maxumum number of requests to handle (-1 == infinite) - opreg_svc.max_reqs = -1; - - log_debug(opreg_debug_level, "Starting Server: url = %s", url.c_str()); - - // Tell the NSSI server to output log data - //rpc_debug_level = opreg_debug_level; - - // start processing requests, the client will send a request to exit when done - rc = nssi_service_start(&opreg_svc); - if (rc != NSSI_OK) { - log_info(opreg_debug_level, "exited opreg_svc: %s", - nssi_err_str(rc)); - } - - sleep(5); - - /* shutdown the opreg_svc */ - log_debug(opreg_debug_level, "shutting down service library"); - nssi_service_fini(&opreg_svc); - - - return rc; -} - -/** - * @} - */ diff --git a/packages/trios/tests/opreg/opreg_debug.cpp b/packages/trios/tests/opreg/opreg_debug.cpp deleted file mode 100644 index 94c1dd6daf92..000000000000 --- a/packages/trios/tests/opreg/opreg_debug.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * opreg_debug.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -#include "Trios_logger.h" -#include "opreg_debug.h" - -/** - * Initialize the opreg_debug_level variable to LOG_UNDEFINED. This - * tells the logger to use the value set by logger_init, if called. - */ -log_level opreg_debug_level = LOG_UNDEFINED; diff --git a/packages/trios/tests/opreg/opreg_debug.h b/packages/trios/tests/opreg/opreg_debug.h deleted file mode 100644 index 0727e4cdc5ac..000000000000 --- a/packages/trios/tests/opreg/opreg_debug.h +++ /dev/null @@ -1,57 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * opreg_debug.h - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -#ifndef OPREG_DEBUG_H_ -#define OPREG_DEBUG_H_ - -#include "Trios_logger.h" - -extern log_level opreg_debug_level; - -#endif /* OPREG_CLIENT_H_ */ diff --git a/packages/trios/tests/opreg/opreg_service_args.x b/packages/trios/tests/opreg/opreg_service_args.x deleted file mode 100644 index 96b99b37d5c9..000000000000 --- a/packages/trios/tests/opreg/opreg_service_args.x +++ /dev/null @@ -1,127 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/** - * @file opreg_service_args.x - * - * @brief Type definitions for an request opreg test. - * - * @author Todd kordenbrock (thkorde\@sandia.gov). - * - * Created on: Aug 22, 2011 - * - */ - -/** - * @defgroup opreg_example Nessie Data Transfer Example - * - * The data-transfer example demonstrates a simple client - * and server that transfer an array of 16-byte \ref data_t - * data structures from a parallel application to a set of - * servers. We implemented two variations: - * one that transfers the array of data structures - * with the request, and a second method that has each server - * pull the data using the \ref nssi_get_data() function. Although - * this example is fairly simple, it makes a decent benchmark code - * to evaluate overheads of the Nessie transfer protocols and encoding - * schemes. - * -*/ - -/** - * @defgroup opreg_types Nessie Example Types - * @ingroup opreg_example - * - * @{ - */ - -/* Extra stuff to put at the beginning of the header file */ -#ifdef RPC_HDR -%#include "Trios_xdr.h" -#endif - -/* Extra stuff to put at the beginning of the C file. */ -#ifdef RPC_XDR -%#include "Trios_xdr.h" -#endif - - - -enum opreg_test_type { - OPREG_C_TEST = 0, - OPREG_CPP_TEST -}; - -/** - * @brief Opcodes for the types of transfer operations. - */ -enum opreg_op { - /** Opcode for sending a request. */ - OPREG_REQUEST_OP = 1 -}; - -/** - * @brief A 16-byte structure that contains an int, float, and double. - * - * This structure contains an int, float, and double as an example - * of a complex structure with multiple types. This will exercise the - * encoding/decoding features of Nessie. - */ -struct data_t { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -}; - -struct opreg_args { - /* test data */ - data_t data; - /** 32-bit checksum of the test data */ - uint64_t chksum; -}; - -/** - * @} - */ diff --git a/packages/trios/tests/opreg/opreg_test.cpp b/packages/trios/tests/opreg/opreg_test.cpp deleted file mode 100644 index 1bd44bbc54d2..000000000000 --- a/packages/trios/tests/opreg/opreg_test.cpp +++ /dev/null @@ -1,569 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * opreg_service_test.cpp - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - - -#include "Trios_config.h" -#include "Trios_nssi_client.h" -#include "Trios_nssi_xdr.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include "opreg_test.h" -#include "opreg_service_args.h" -#include "opreg_debug.h" - - -#include -#include -#include - -#include -#include -#include - - -// Prototypes for client and server codes -extern "C" { -uint64_t calc_checksum (const char * buf, const uint64_t size); -int opreg_c_server_main(nssi_rpc_transport transport, MPI_Comm server_comm); -} -int opreg_cpp_server_main(nssi_rpc_transport transport, MPI_Comm server_comm); -int opreg_client_main (struct opreg_cmdline_args &args, nssi_service &opreg_svc, MPI_Comm client_comm); - - -/* -------------- private methods -------------------*/ - -uint64_t calc_checksum (const char * buf, const uint64_t size) -{ - unsigned long hash = 5381; - const char* p = buf; - uint64_t i = 0; - - for (i=0; i -1) { - args.transport =nssi_transport_list[transport_index]; - args.transport_name=std::string(nssi_transport_names[transport_index]); - } - args.opreg_type_name=opreg_type_names[args.opreg_type]; - - log_debug(args.debug_level, "%d: Finished processing arguments", rank); - - - if (!success) { - MPI_Abort(MPI_COMM_WORLD, 1); - } - - - if (!args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.client.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && !args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.server.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } else if (args.server_flag && args.client_flag) { - /* initialize logger */ - if (args.logfile.empty()) { - logger_init(args.debug_level, NULL); - } else { - char fn[1024]; - sprintf(fn, "%s.%03d.log", args.logfile.c_str(), rank); - logger_init(args.debug_level, fn); - } - } - - log_level debug_level = args.debug_level; - - // Communicator used for both client and server (may split if using client and server) - MPI_Comm comm; - - log_debug(debug_level, "%d: Starting opreg-service test", rank); - - /** - * Since this test can be run as a server, client, or both, we need to play some fancy - * MPI games to get the communicators working correctly. If we're executing as both - * a client and a server, we split the communicator so that the client thinks its - * running by itself. - */ - if (args.client_flag && args.server_flag) { - if (np < 2) { - log_error(debug_level, "Must use at least 2 MPI processes for client and server mode"); - MPI_Abort(MPI_COMM_WORLD, -1); - } - - // Split the communicators. Processors with color=0 are servers. - - int color = (rank == 0) ? 0 : 1; // only one server - MPI_Comm_split(MPI_COMM_WORLD, color, rank, &comm); - - MPI_Comm_rank(comm, &splitrank); - MPI_Comm_size(comm, &splitsize); - - // std::cout << "rank=" << rank << "/" << np << ", color=" << color << - // ", new_rank=" << newrank << "/" << newsize << std::endl << std::endl; - // - // std::cout << "my_url=" << my_url << ", server_url=" << args.server_url << std::endl; - } - else { - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - } - - /** - * Initialize the Nessie interface by specifying a transport, encoding scheme, and a - * recommended URL. \ref NSSI_DEFAULT_TRANSPORT is usually the best choice, since it - * is often the case that only one type of transport exists on a particular platform. - * Currently supported transports are \ref NSSI_RPC_PTL, \ref NSSI_RPC_GNI, and - * \ref NSSI_RPC_IB. We only support one type of encoding scheme so NSSI_DEFAULT_ENCODE - * should always be used for the second argument. The URL can be specified (as we did for - * the server, or NULL (as we did for the client). This is a recommended value. Use the - * \ref nssi_get_url function to find the actual value. - */ - if (args.server_flag && !args.server_url.empty()) { - // use the server URL as suggested URL - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, args.server_url.c_str()); - } - else { - nssi_rpc_init((nssi_rpc_transport)args.transport, NSSI_DEFAULT_ENCODE, NULL); - } - - // Get the Server URL - std::string my_url(NSSI_URL_LEN, '\0'); - nssi_get_url((nssi_rpc_transport)args.transport, &my_url[0], NSSI_URL_LEN); - - // Broadcast the server URL to all the clients - args.server_url.resize(NSSI_URL_LEN, '\0'); - if (args.server_flag && args.client_flag) { - args.server_url = my_url; - MPI_Bcast(&args.server_url[0], args.server_url.size(), MPI_CHAR, 0, MPI_COMM_WORLD); - } - - else if (!args.server_flag && args.client_flag){ - if (args.server_url.empty()) { - - // check to see if we're supposed to get the URL from a file - if (!args.url_file.empty()) { - // Fetch the server URL from a file - sleep(1); - log_debug(debug_level, "Reading from file %s", args.url_file.c_str()); - std::ifstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - if (urlfile.good()) - getline(urlfile, args.server_url); - } - else { - log_error(debug_level, "Failed to open server_url_file=%s", args.url_file.c_str()); - exit(1); - } - urlfile.close(); - log_debug(debug_level, "URL = %s", args.server_url.c_str()); - } - else { - log_error(debug_level, "Need to set --server-url=[ADDR] or --server-url-file=[PATH]"); - } - } - } - - else if (args.server_flag && !args.client_flag) { - args.server_url = my_url; - - // If the url_file value is set, write the url to a file - if (!args.url_file.empty()) { - std::ofstream urlfile (args.url_file.c_str()); - if (urlfile.is_open()) { - urlfile << args.server_url.c_str() << std::endl; - } - urlfile.close(); - log_debug(debug_level, "Wrote url to file %s", args.url_file.c_str()); - } - } - - - - // Set the debug level for the opreg service. - opreg_debug_level = args.debug_level; - - // Print the arguments after they've all been set. - print_args(out, args, "%"); - - - //------------------------------------------------------------------------------ - /** If we're running this job with a server, the server always executes on node 0. - * In this example, the server is a single process. - */ - if (args.server_flag && (rank == 0)) { - if (args.opreg_type == OPREG_C_TEST) { - rc = opreg_c_server_main((nssi_rpc_transport)args.transport, comm); - } else if (args.opreg_type == OPREG_CPP_TEST) { - rc = opreg_cpp_server_main((nssi_rpc_transport)args.transport, comm); - } - log_debug(debug_level, "Server is finished"); - } - - // ------------------------------------------------------------------------------ - /** The parallel client will execute this branch. The root node, node 0, of the client connects - * connects with the server, using the \ref nssi_get_service function. Then the root - * broadcasts the service description to the other clients before starting the main - * loop of the client code by calling \ref opreg_client_main. - */ - else { - int i; - int client_rank; - - // get rank within the client communicator - MPI_Comm_rank(comm, &client_rank); - - nssi_init((nssi_rpc_transport)args.transport); - - // Only one process needs to connect to the service - // TODO: Make get_service a collective call (some transports do not need a connection) - //if (client_rank == 0) { - { - - sleep(args.delay); // give server time to get started - - // connect to remote server - for (i=0; i < args.num_retries; i++) { - log_debug(debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service((nssi_rpc_transport)args.transport, args.server_url.c_str(), args.timeout, &opreg_svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(opreg_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - } - - //MPI_Bcast(&rc, 1, MPI_INT, 0, comm); - - if (rc == NSSI_OK) { - if (client_rank == 0) log_debug(debug_level, "Connected to service on attempt %d\n", i); - - // Broadcast the service description to the other clients - //log_debug(opreg_debug_level, "Bcasting svc to other clients"); - //MPI_Bcast(&opreg_svc, sizeof(nssi_service), MPI_BYTE, 0, comm); - - log_debug(debug_level, "Starting client main"); - // Start the client code - opreg_client_main(args, opreg_svc, comm); - - - MPI_Barrier(comm); - - // Tell one of the clients to kill the server - if (client_rank == 0) { - log_debug(debug_level, "%d: Halting opreg service", rank); - rc = nssi_kill(&opreg_svc, 0, 5000); - } - } - - else { - if (client_rank == 0) - log_error(debug_level, "Failed to connect to service after %d attempts: ABORTING", i); - success = false; - //MPI_Abort(MPI_COMM_WORLD, -1); - } - - nssi_fini((nssi_rpc_transport)args.transport); - - } - - log_debug(debug_level, "%d: clean up nssi", rank); - MPI_Barrier(MPI_COMM_WORLD); - - // Clean up nssi_rpc - rc = nssi_rpc_fini((nssi_rpc_transport)args.transport); - if (rc != NSSI_OK) - log_error(debug_level, "Error in nssi_rpc_fini"); - - log_debug(debug_level, "%d: MPI_Finalize()", rank); - MPI_Finalize(); - - logger_fini(); - - if(success && (rc == NSSI_OK)) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return ((success && (rc==NSSI_OK)) ? 0 : 1 ); -} diff --git a/packages/trios/tests/opreg/opreg_test.h b/packages/trios/tests/opreg/opreg_test.h deleted file mode 100644 index 0c224e626e5e..000000000000 --- a/packages/trios/tests/opreg/opreg_test.h +++ /dev/null @@ -1,100 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * opreg_client.h - * - * Created on: Nov 14, 2011 - * Author: thkorde - */ - -#ifndef OPREG_CLIENT_H_ -#define OPREG_CLIENT_H_ - -#include -#include - -#include "Trios_logger.h" -#include "Trios_nssi_fprint_types.h" -#include "Trios_nnti_fprint_types.h" - - -/** - * Options and arguments passed to the client driver. - */ -struct opreg_cmdline_args { - bool client_flag; - bool server_flag; - int transport; - std::string transport_name; - int opreg_type; - std::string server_url; - std::string url_file; - std::string opreg_type_name; - log_level debug_level; - std::string logfile; - int timeout; - int delay; - int num_retries; -}; - - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(__STDC__) || defined(__cplusplus) - - - -#else /* K&R C */ -#endif - - - - -#ifdef __cplusplus -} -#endif - - -#endif /* OPREG_CLIENT_H_ */ diff --git a/packages/trios/tests/unit/AtomicsTest.cpp b/packages/trios/tests/unit/AtomicsTest.cpp deleted file mode 100644 index 6c2b70d93b68..000000000000 --- a/packages/trios/tests/unit/AtomicsTest.cpp +++ /dev/null @@ -1,408 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * NssiWaitTest.c - * - * Created on: August 14, 2014 - * Author: thkorde - */ - -#include "Trios_nssi_client.h" -#include "Trios_nssi_server.h" -#include "Trios_nssi_fprint_types.h" - -#include "Trios_nnti.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include - -#include - -#include - -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_trace.h" - -#include "nssi_opcodes.h" -#include "nssi_service_args.h" - -#include -#include - -extern NNTI_transport_t transports[NSSI_RPC_COUNT]; - -log_level atomics_debug_level = LOG_UNDEFINED; - - -#define REQ_COUNT 5 - - -int fetchadd_test(const nssi_service *svc) -{ - int rc = NNTI_OK; - int timeout = 5000; - - NNTI_work_request_t wr; - NNTI_status_t status; - - int64_t value=-1; - - log_debug(atomics_debug_level, "enter"); - - for (int varid=0;varid<10;varid++) { - for (int i=0;i<10;i++) { - rc=NNTI_atomic_fop( - &transports[svc->transport_id], - &svc->svc_host, - varid, - varid, - 1, - NNTI_ATOMIC_FADD, - &wr); - /* wait for completion */ - NNTI_wait(&wr, timeout, &status); - if (rc != NNTI_OK) { - log_error(atomics_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - - rc=NNTI_atomic_read( - &transports[svc->transport_id], - varid, - &value); - - if (value != i) { - log_error(atomics_debug_level, "actual=%d, expected=%d", value, i); - } - - log_debug(atomics_debug_level, "varid=%03d value=%lld", varid, value); - } - - for (int i=10;i>0;i--) { - rc=NNTI_atomic_fop( - &transports[svc->transport_id], - &svc->svc_host, - varid, - varid, - -1, - NNTI_ATOMIC_FADD, - &wr); - /* wait for completion */ - NNTI_wait(&wr, timeout, &status); - if (rc != NNTI_OK) { - log_error(atomics_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - - rc=NNTI_atomic_read( - &transports[svc->transport_id], - varid, - &value); - - if (value != i) { - log_error(atomics_debug_level, "actual=%d, expected=%d", value, i); - } - - log_debug(atomics_debug_level, "varid=%03d value=%lld", varid, value); - } - } - - log_debug(atomics_debug_level, "exit"); - - return rc; -} - -int cswap_test(const nssi_service *svc) -{ - int rc = NNTI_OK; - int timeout = 5000; - - NNTI_work_request_t wr; - NNTI_status_t status; - - int64_t cmp_target=5; - int64_t value=-1; - - log_debug(atomics_debug_level, "enter"); - - for (int varid=0;varid<10;varid++) { - for (int i=0;i<10;i++) { - rc=NNTI_atomic_fop( - &transports[svc->transport_id], - &svc->svc_host, - varid, - varid, - 1, - NNTI_ATOMIC_FADD, - &wr); - /* wait for completion */ - rc=NNTI_wait(&wr, timeout, &status); - if (rc != NNTI_OK) { - log_error(atomics_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - - rc=NNTI_atomic_read( - &transports[svc->transport_id], - varid, - &value); - - log_debug(atomics_debug_level, "after fetch-add varid=%03d value=%lld", varid, value); - - if (value != (i%cmp_target)) { - log_error(atomics_debug_level, "actual=%d, expected=%d", value, (i%cmp_target)); - } - - rc=NNTI_atomic_cswap( - &transports[svc->transport_id], - &svc->svc_host, - varid, - varid, - cmp_target, - 0, - &wr); - /* wait for completion */ - rc=NNTI_wait(&wr, timeout, &status); - if (rc != NNTI_OK) { - log_error(atomics_debug_level, "remote method failed: %s", - nnti_err_str(rc)); - return rc; - } - - rc=NNTI_atomic_read( - &transports[svc->transport_id], - varid, - &value); - - log_debug(atomics_debug_level, "after compare-swap varid=%03d value=%lld", varid, value); - - int64_t actual=value; - int64_t expected=(i+1)%cmp_target; - - if (i%cmp_target == cmp_target-1) { - if (actual != cmp_target) { - log_error(atomics_debug_level, "actual=%d, expected=%d", actual, expected); - } - } else { - if (actual != expected) { - log_error(atomics_debug_level, "actual=%d, expected=%d", actual, expected); - } - } - } - } - - log_debug(atomics_debug_level, "exit"); - - return rc; -} - -int nssi_atomics_test(const nssi_service *svc) -{ - int rc = NNTI_OK; - int timeout = 5000; - - NNTI_work_request_t wr; - NNTI_status_t status; - - int64_t value=-1; - - log_debug(atomics_debug_level, "enter"); - - for (int varid=0;varid<10;varid++) { - for (int i=0;i<10;i++) { - rc=nssi_atomic_increment( - svc, - varid, - varid); - - rc=nssi_atomic_read( - svc, - varid, - &value); - - if (value != i) { - log_error(atomics_debug_level, "actual=%d, expected=%d", value, i); - } - - log_debug(atomics_debug_level, "varid=%03d value=%lld", varid, value); - } - - for (int i=10;i>0;i--) { - rc=nssi_atomic_decrement( - svc, - varid, - varid); - - rc=nssi_atomic_read( - svc, - varid, - &value); - - if (value != i) { - log_error(atomics_debug_level, "actual=%d, expected=%d", value, i); - } - - log_debug(atomics_debug_level, "varid=%03d value=%lld", varid, value); - } - } - - log_debug(atomics_debug_level, "exit"); - - return rc; -} - - -int main(int argc, char *argv[]) -{ - int test_result=0; - int rc=0; - int success=0; - int nprocs, rank; - - char logname[1024]; - char url[NSSI_URL_LEN]; - - nssi_service svc; - - - MPI_Init(&argc, &argv); - - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - - struct sigaction sigact; - sigact.sa_handler=SIG_DFL; - sigaction ( 6, &sigact, NULL); - sigaction (11, &sigact, NULL); - - sprintf(logname, "atomics.%03d.log", rank); - logger_init(LOG_ERROR, NULL); - - // common init - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - if (rank==0) { - // service init - nssi_service_init(NSSI_DEFAULT_TRANSPORT, NSSI_SHORT_REQUEST_SIZE, &svc); - nssi_get_url(NSSI_DEFAULT_TRANSPORT, &url[0], NSSI_URL_LEN); - - MPI_Bcast(&url[0], NSSI_URL_LEN, MPI_CHAR, 0, MPI_COMM_WORLD); - - svc.max_reqs = -1; - rc = nssi_service_start(&svc); - if (rc != NSSI_OK) { - log_info(atomics_debug_level, "exited selfsend_svc: %s", - nssi_err_str(rc)); - } - - /* finalize the service */ - nssi_service_fini(&svc); - - } else { - // client init - nssi_init(NSSI_DEFAULT_TRANSPORT); - - MPI_Bcast(&url[0], NSSI_URL_LEN, MPI_CHAR, 0, MPI_COMM_WORLD); - - for (int i=0; i < 3; i++) { - log_debug(atomics_debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service(NSSI_DEFAULT_TRANSPORT, url, 5000, &svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(atomics_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - - test_result=fetchadd_test(&svc); - if (test_result==NNTI_OK) { - test_result=cswap_test(&svc); - } - if (test_result==NNTI_OK) { - test_result=nssi_atomics_test(&svc); - } - - // shutdown the service - rc = nssi_kill(&svc, 0, 5000); - if (rc != NSSI_OK) { - log_error(atomics_debug_level, "Error in nssi_kill"); - } - - // finalize the client - nssi_fini(NSSI_DEFAULT_TRANSPORT); - - if (test_result == NNTI_OK) { - success=1; - } else { - success=0; - } - } - - // finalize the NSSI library - rc = nssi_rpc_fini(NSSI_DEFAULT_TRANSPORT); - if (rc != NSSI_OK) { - log_error(atomics_debug_level, "Error in nssi_rpc_fini"); - } - - MPI_Bcast(&success, 1, MPI_INT, 1, MPI_COMM_WORLD); - - MPI_Finalize(); - - logger_fini(); - - if (rank == 1) { - if (success==TRUE) - fprintf(stdout, "\nEnd Result: TEST PASSED\n"); - else - fprintf(stdout, "\nEnd Result: TEST FAILED\n"); - } - - return (success==TRUE ? 0 : 1 ); -} diff --git a/packages/trios/tests/unit/CMakeLists.txt b/packages/trios/tests/unit/CMakeLists.txt deleted file mode 100644 index 2c3a38f3c5c5..000000000000 --- a/packages/trios/tests/unit/CMakeLists.txt +++ /dev/null @@ -1,212 +0,0 @@ - -# @HEADER -# ************************************************************************ -# -# Trios: Trilinos I/O Support -# Copyright 2011 Sandia Corporation -# -# Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -# the U.S. Government retains certain rights in this software. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the Corporation nor the names of the -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -# Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -# -# ************************************************************************* -# @HEADER - - - -# -# When adding test directories if you want the test to be distributed -# simply add the subdirectory like normal. If you do not want the test -# to be distributed then you need to test to add it like: -# IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/) -# ADD_SUBDIRECTORY() -# ENDIF() -# -# This will prevent configure errors from not being able to find the -# test's directory when it has been removed from distributions -# - - -#IF(PERL_EXECUTABLE) -# SET(ZOLTAN_TEST_DRIVER CMND ${PERL_EXECUTABLE} ARGS ../ctest_zoltan.pl) -#ELSE() -# MESSAGE(STATUS "Skipping Zoltan tests because PERL_EXECUTABLE is not found.") -# RETURN() -#ENDIF() - -#IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ch_brack2_3) -# ADD_SUBDIRECTORY(ch_brack2_3) -#ENDIF() - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - AtomicsTest - SOURCES AtomicsTest.cpp - COMM serial mpi - NUM_MPI_PROCS 2 - NOEXEPREFIX -) - -IF (TPL_ENABLE_MPI) - TRIBITS_ADD_EXECUTABLE( - FullQueueTest - SOURCES FullQueueTest.cpp - COMM serial mpi -# NUM_MPI_PROCS 2 - NOEXEPREFIX - ) -ENDIF (TPL_ENABLE_MPI) - - -IF (TPL_ENABLE_Pthread) - IF (CMAKE_HAVE_PTHREAD_H) - TRIBITS_ADD_EXECUTABLE( - InterruptTest - SOURCES InterruptTest.c - COMM serial mpi -# NUM_MPI_PROCS 1 - NOEXEPREFIX - ) - ENDIF (CMAKE_HAVE_PTHREAD_H) -ENDIF (TPL_ENABLE_Pthread) - - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - LoggerTest - SOURCES LoggerTest.cpp - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - NntiPackUnpackLocalTest - SOURCES NntiPackUnpackLocalTest.cpp - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE( - NntiPerfTest - SOURCES NntiPerfTest.cpp -# NUM_MPI_PROCS 2 - NOEXEPREFIX -) - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - NntiSelfSendTest - SOURCES NntiSelfSendTest.cpp - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX -) - - -IF (TPL_ENABLE_MPI) - TRIBITS_ADD_EXECUTABLE( - NonContigTest - SOURCES NonContigTest.cpp - COMM serial mpi -# NUM_MPI_PROCS 2 - NOEXEPREFIX - ) -ENDIF (TPL_ENABLE_MPI) - - -IF (TPL_ENABLE_MPI) - TRIBITS_ADD_EXECUTABLE( - MultiUseBufferTest - SOURCES MultiUseBufferTest.cpp - COMM serial mpi -# NUM_MPI_PROCS 2 - NOEXEPREFIX - ) -ENDIF (TPL_ENABLE_MPI) - - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - NssiSelfSendTest - SOURCES NssiSelfSendTest.cpp - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX -) - - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - NssiWaitTest - SOURCES NssiWaitTest.cpp - COMM serial mpi - NUM_MPI_PROCS 2 - NOEXEPREFIX -) - - -IF (TPL_ENABLE_Pthread) - IF (CMAKE_HAVE_PTHREAD_H) - TRIBITS_ADD_EXECUTABLE( - ProgressTimeoutTest - SOURCES ProgressTimeoutTest.cpp - COMM serial mpi -# NUM_MPI_PROCS 1 - NOEXEPREFIX - ) - ENDIF (CMAKE_HAVE_PTHREAD_H) -ENDIF (TPL_ENABLE_Pthread) - - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - SignalTest - SOURCES SignalTest.c - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX -) - - -TRIBITS_ADD_EXECUTABLE_AND_TEST( - TimerTest - SOURCES TimerTest.c - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX -) - - -IF (TRIOS_ENABLE_TRACING) - TRIBITS_ADD_EXECUTABLE_AND_TEST( - TraceTest - SOURCES TraceTest.c - COMM serial mpi - NUM_MPI_PROCS 1 - NOEXEPREFIX - ) -ENDIF (TRIOS_ENABLE_TRACING) diff --git a/packages/trios/tests/unit/FullQueueTest.cpp b/packages/trios/tests/unit/FullQueueTest.cpp deleted file mode 100644 index 78908f774551..000000000000 --- a/packages/trios/tests/unit/FullQueueTest.cpp +++ /dev/null @@ -1,642 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * FullQueueTest.c - * - * Created on: March 24, 2014 - * Author: thkorde - */ - -#include "Trios_nnti.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nnti_fprint_types.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include - -#include - - - -#define SYNC_WITH_BARRIER - - -#define QUEUE_SIZE 10 - - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; -char url[NNTI_URL_LEN]; - -NNTI_buffer_t queue_mr; -NNTI_work_request_t queue_wr; - -NNTI_buffer_t client_ack_mr; -NNTI_work_request_t client_ack_wr; - -NNTI_buffer_t server_ack_mr; -NNTI_work_request_t server_ack_wr; - -NNTI_buffer_t send_mr; /* contiguous registered memory regions */ -NNTI_work_request_t send_wr[QUEUE_SIZE]; /* work requests */ -NNTI_work_request_t *send_wr_list[QUEUE_SIZE]; /* work requests */ - -NNTI_work_request_t send_ack_wr; - -#define WR_COUNT 2 -NNTI_work_request_t *mr_wr_list[WR_COUNT]; -NNTI_result_t err; - -int one_mb=1024*1024; - -log_level fullqueue_debug_level = LOG_UNDEFINED; - -int sent_count =0; -int success_count=0; -int dropped_count=0; -int timeout_count=0; -int fill_count =0; - -static int buffer_pack(void *input, char **output, uint64_t *output_size) -{ - NNTI_dt_sizeof(&trans_hdl, input, output_size); - *output=(char*)malloc(*output_size); - NNTI_dt_pack(&trans_hdl, input, *output, *output_size); - - return(0); -} - -static int buffer_free(void *input) -{ - NNTI_dt_free(&trans_hdl, input); - - return(0); -} - -static int buffer_unpack(char *input, uint64_t input_size, void *output) -{ - NNTI_dt_unpack(&trans_hdl, output, input, input_size); - - return(0); -} - -void client(void) { - NNTI_result_t rc=NNTI_OK; - NNTI_status_t send_status[QUEUE_SIZE]; - NNTI_status_t *send_status_list[QUEUE_SIZE]; - NNTI_status_t client_ack_status; - char *c_ptr; - char *packed=NULL; - uint64_t packed_size=0; - - - NNTI_connect(&trans_hdl, url, 5000, &server_hdl); - - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, &send_mr); - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_RECV_DST, &client_ack_mr); - - log_debug(LOG_ALL, "Client Begin Phase #1 - exchange ACK buffers"); - /* - * Phase 1 - exchange ACK buffers - */ - buffer_pack(&client_ack_mr, &packed, &packed_size); - if (packed_size > NNTI_REQUEST_BUFFER_SIZE) { - log_error(fullqueue_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); - MPI_Abort(MPI_COMM_WORLD, -10); - } - - // send the server the recv_mr so it can send back it's ack_mr - c_ptr=NNTI_BUFFER_C_POINTER(&send_mr); - memcpy(c_ptr, packed, packed_size); - - rc=NNTI_send(&server_hdl, &send_mr, NULL, &send_wr[0]); - if (rc == NNTI_OK) { - sent_count++; - } else { - log_error(fullqueue_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_wr[0], 1000, &send_status[0]); - if (rc == NNTI_OK) { - success_count++; - } else { - log_error(fullqueue_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - - // wait for the server to send back it's recv_mr - NNTI_create_work_request(&client_ack_mr, &client_ack_wr); - NNTI_wait(&client_ack_wr, -1, &client_ack_status); - - c_ptr=(char*)client_ack_status.start+client_ack_status.offset; - buffer_unpack(c_ptr, client_ack_status.length, &server_ack_mr); - - NNTI_destroy_work_request(&client_ack_wr); - -// fprint_NNTI_buffer(logger_get_file(), "server_ack_mr", -// "received server ack hdl", &server_ack_mr); - -#ifdef SYNC_WITH_BARRIER - MPI_Barrier(MPI_COMM_WORLD); -#else - // send an ACK so the server knows to proceed - rc=NNTI_send(&server_hdl, &send_mr, &server_ack_mr, &send_wr[0]); - if (rc != NNTI_OK) { - log_error(fullqueue_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_wr[0], 1000, &send_status[0]); - if (rc != NNTI_OK) { - log_error(fullqueue_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } -#endif - - - log_debug(LOG_ALL, "Client Begin Phase #2 - the client fills the queue"); - /* - * Phase 2 - fill the server's queue - */ - // the server's queue has QUEUE_SIZE slots available, so these NNTI_send() operations should all succeed. - while (1) { - rc=NNTI_send(&server_hdl, &send_mr, NULL, &send_wr[0]); - if (rc == NNTI_OK) { - sent_count++; - } else { - log_error(fullqueue_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_wr[0], 1000, &send_status[0]); - if (rc == NNTI_OK) { - success_count++; - fill_count++; - } else if (rc == NNTI_EDROPPED) { - dropped_count++; - break; - } else if (rc == NNTI_ETIMEDOUT) { - timeout_count++; - break; - } else { - log_error(fullqueue_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - } - - c_ptr=NNTI_BUFFER_C_POINTER(&send_mr); - *(int*)c_ptr=fill_count; - - // send an ACK so the server knows to proceed - rc=NNTI_send(&server_hdl, &send_mr, &server_ack_mr, &send_ack_wr); - if (rc != NNTI_OK) { - log_error(fullqueue_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_ack_wr, 1000, &send_status[0]); - if (rc != NNTI_OK) { - log_error(fullqueue_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - - log_debug(LOG_ALL, "Client Begin Phase #3 - the client sends to a full queue (blocked_count=%d)", QUEUE_SIZE); - /* - * Phase 3 - send to a full queue - */ - MPI_Barrier(MPI_COMM_WORLD); - - send_wr_list[0]=&send_wr[0]; - send_status_list[0]=&send_status[0]; - // the server's queue should be full, so these NNTI_send() operations should fail with NNTI_ETIMEDOUT. - for (int i=1;i NNTI_REQUEST_BUFFER_SIZE) { - log_error(fullqueue_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); - MPI_Abort(MPI_COMM_WORLD, -10); - } - - // send our recv_mr back to the client - c_ptr=NNTI_BUFFER_C_POINTER(&send_mr); - memcpy(c_ptr, packed, packed_size); - - rc=NNTI_send(&queue_status.src, &send_mr, &client_ack_mr, &send_wr[0]); - if (rc != NNTI_OK) { - log_error(fullqueue_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_wr[0], -1, &send_status[0]); - if (rc != NNTI_OK) { - log_error(fullqueue_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - -#ifdef SYNC_WITH_BARRIER - MPI_Barrier(MPI_COMM_WORLD); -#else - NNTI_create_work_request(&server_ack_mr, &server_ack_wr); - NNTI_wait(&server_ack_wr, -1, &server_ack_status); - NNTI_destroy_work_request(&server_ack_wr); -#endif - - - log_debug(LOG_ALL, "Server Begin Phase #2 - the client fills the queue"); - - /* - * Phase 2 - the client fills the queue - */ - NNTI_create_work_request(&server_ack_mr, &server_ack_wr); - NNTI_wait(&server_ack_wr, -1, &server_ack_status); - NNTI_destroy_work_request(&server_ack_wr); - - fill_count=*(int*)((char*)server_ack_status.start+server_ack_status.offset); - - - log_debug(LOG_ALL, "Server Begin Phase #3 - the client sends to a full queue"); - /* - * Phase 3 - the client sends to the full queue - */ - MPI_Barrier(MPI_COMM_WORLD); - - MPI_Barrier(MPI_COMM_WORLD); - - - log_debug(LOG_ALL, "Server Begin Phase #4 - the server drains the queue (requests sent to empty queue) (fill_count=%d)", fill_count); - /* - * Phase 4 - drain the queue - */ - for (int i=0;i -#include - -#include "Trios_logger.h" -#include "Trios_timer.h" - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; -NNTI_buffer_t mr1, mr2, mr3; /* registered memory regions */ -NNTI_work_request_t mr1_wr, mr2_wr, mr3_wr; /* work requests */ -#define WR_COUNT 3 -NNTI_work_request_t *mr_wr_list[WR_COUNT]; -NNTI_result_t err; -NNTI_status_t status; - -log_level interrupt_debug_level=LOG_UNDEFINED; - -int expected_wait_time=0; -int actual_wait_time=0; - -pthread_barrier_t barrier; - -static int time_to_exit=FALSE; -static void *do_wait(void *args) -{ - NNTI_result_t rc; - NNTI_status_t wait_status1; - NNTI_status_t wait_status2; - NNTI_status_t wait_status3; - NNTI_status_t *wait_status_list[WR_COUNT]; - - uint32_t which=0; - - double wait_time=0.0; - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_wait() #1 won't be interrupted so should timeout after 10000 ms."); - - wait_time=trios_get_time_ms(); - rc=NNTI_wait(&mr1_wr, 10000, &wait_status1); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_wait() #1: %lums", (uint64_t)wait_time); - - expected_wait_time+=10000; - actual_wait_time+=wait_time; - - pthread_barrier_wait(&barrier); - /* between these barriers the main thread is calling NNTI_interrupt() so that the next NNTI_wait() returns immediately. */ - pthread_barrier_wait(&barrier); - - log_debug(interrupt_debug_level, "NNTI_wait() #2 should return immediately."); - - wait_time=trios_get_time_ms(); - rc=NNTI_wait(&mr2_wr, 10000, &wait_status2); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_wait() #2: %lums", (uint64_t)wait_time); - - expected_wait_time+=0; - actual_wait_time+=wait_time; - - pthread_barrier_wait(&barrier); - - log_debug(interrupt_debug_level, "NNTI_wait() #3 should return after 3000 ms."); - - wait_time=trios_get_time_ms(); - rc=NNTI_wait(&mr3_wr, 10000, &wait_status3); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_wait() #3: %lums", (uint64_t)wait_time); - - expected_wait_time+=3000; - actual_wait_time+=wait_time; - - /* between these barriers the main thread is sleeping 3 seconds and then calling NNTI_interrupt(). */ - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_waitany() #1 won't be interrupted so should timeout after 10000 ms."); - - mr_wr_list[0]=&mr1_wr; - mr_wr_list[1]=&mr2_wr; - mr_wr_list[2]=&mr3_wr; - wait_time=trios_get_time_ms(); - rc=NNTI_waitany(mr_wr_list, WR_COUNT, 10000, &which, &wait_status1); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_waitany() #1: %lums", (uint64_t)wait_time); - - expected_wait_time+=10000; - actual_wait_time+=wait_time; - - pthread_barrier_wait(&barrier); - /* between these barriers the main thread is calling NNTI_interrupt() so that the next NNTI_waitany() returns immediately. */ - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_waitany() #2 should return immediately."); - - mr_wr_list[0]=&mr1_wr; - mr_wr_list[1]=&mr2_wr; - mr_wr_list[2]=&mr3_wr; - wait_time=trios_get_time_ms(); - rc=NNTI_waitany(mr_wr_list, WR_COUNT, 10000, &which, &wait_status2); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_waitany() #2: %lums", (uint64_t)wait_time); - - expected_wait_time+=0; - actual_wait_time+=wait_time; - - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_waitany() #3 should return after 3000 ms."); - - mr_wr_list[0]=&mr1_wr; - mr_wr_list[1]=&mr2_wr; - mr_wr_list[2]=&mr3_wr; - wait_time=trios_get_time_ms(); - rc=NNTI_waitany(mr_wr_list, WR_COUNT, 10000, &which, &wait_status3); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_waitany() #3: %lums", (uint64_t)wait_time); - - expected_wait_time+=3000; - actual_wait_time+=wait_time; - - /* between these barriers the main thread is sleeping 3 seconds and then calling NNTI_interrupt(). */ - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_waitall() #1 won't be interrupted so should timeout after 10000 ms."); - - mr_wr_list[0]=&mr1_wr; - mr_wr_list[1]=&mr2_wr; - mr_wr_list[2]=&mr3_wr; - wait_status_list[0]=&wait_status1; - wait_status_list[1]=&wait_status2; - wait_status_list[2]=&wait_status3; - wait_time=trios_get_time_ms(); - rc=NNTI_waitall(mr_wr_list, WR_COUNT, 10000, wait_status_list); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_waitall() #1: %lums", (uint64_t)wait_time); - - expected_wait_time+=10000; - actual_wait_time+=wait_time; - - pthread_barrier_wait(&barrier); - /* between these barriers the main thread is calling NNTI_interrupt() so that the next NNTI_waitall() returns immediately. */ - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_waitall() #2 should return immediately."); - - mr_wr_list[0]=&mr1_wr; - mr_wr_list[1]=&mr2_wr; - mr_wr_list[2]=&mr3_wr; - wait_status_list[0]=&wait_status1; - wait_status_list[1]=&wait_status2; - wait_status_list[2]=&wait_status3; - wait_time=trios_get_time_ms(); - rc=NNTI_waitall(mr_wr_list, WR_COUNT, 10000, wait_status_list); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_waitall() #2: %lums", (uint64_t)wait_time); - - expected_wait_time+=0; - actual_wait_time+=wait_time; - - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - NNTI_create_work_request(&mr1, &mr1_wr); - NNTI_create_work_request(&mr2, &mr2_wr); - NNTI_create_work_request(&mr3, &mr3_wr); - - log_debug(interrupt_debug_level, "NNTI_waitall() #3 should return after 3000 ms."); - - mr_wr_list[0]=&mr1_wr; - mr_wr_list[1]=&mr2_wr; - mr_wr_list[2]=&mr3_wr; - wait_status_list[0]=&wait_status1; - wait_status_list[1]=&wait_status2; - wait_status_list[2]=&wait_status3; - wait_time=trios_get_time_ms(); - rc=NNTI_waitall(mr_wr_list, WR_COUNT, 10000, wait_status_list); - wait_time=trios_get_time_ms()-wait_time; - - log_debug(interrupt_debug_level, "Time to complete NNTI_waitall() #3: %lums", (uint64_t)wait_time); - - expected_wait_time+=3000; - actual_wait_time+=wait_time; - - /* between these barriers the main thread is sleeping 3 seconds and then calling NNTI_interrupt(). */ - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&mr1_wr); - NNTI_destroy_work_request(&mr2_wr); - NNTI_destroy_work_request(&mr3_wr); - - return(NULL); -} - -static pthread_t wait_thread; -static void launch_wait_thread() -{ - /* Start up polling thread */ - fprintf (stdout, "Start wait thread.\n"); - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int rc = pthread_create(&wait_thread, &attr, do_wait, NULL); - if (rc) { - fprintf (stdout, "ERROR: Cannot create polling thread, err code = %d\n", rc); - return; - } - pthread_attr_destroy(&attr); -} -static void join_wait_thread() -{ - void *status; - time_to_exit=TRUE; - int rc = pthread_join(wait_thread, &status); - if (rc) { - fprintf (stdout, "ERROR: Cannot join wait thread, err code = %d\n", rc); - } else { - fprintf (stdout, "Joined wait thread.\n"); - } -} - - -int main(int argc, char *argv[]) -{ - int8_t success=TRUE; - - logger_init(LOG_ERROR, NULL/*"interrupt.log"*/); - - pthread_barrier_init(&barrier, NULL, 2); - - NNTI_init(NNTI_DEFAULT_TRANSPORT, NULL, &trans_hdl); - - NNTI_alloc(&trans_hdl, NNTI_RESULT_BUFFER_SIZE, 1, NNTI_RECV_DST, &mr1); - NNTI_alloc(&trans_hdl, NNTI_RESULT_BUFFER_SIZE, 1, NNTI_RECV_DST, &mr2); - NNTI_alloc(&trans_hdl, NNTI_RESULT_BUFFER_SIZE, 1, NNTI_RECV_DST, &mr3); - - launch_wait_thread(); - - pthread_barrier_wait(&barrier); - NNTI_interrupt(&trans_hdl); - pthread_barrier_wait(&barrier); - /* between these barriers the wait thread is calling NNTI_wait(). */ - pthread_barrier_wait(&barrier); - /* between these barriers the wait thread is calling NNTI_wait(). */ - sleep(3); - NNTI_interrupt(&trans_hdl); - pthread_barrier_wait(&barrier); - - pthread_barrier_wait(&barrier); - NNTI_interrupt(&trans_hdl); - pthread_barrier_wait(&barrier); - /* between these barriers the wait thread is calling NNTI_waitany(). */ - pthread_barrier_wait(&barrier); - /* between these barriers the wait thread is calling NNTI_waitany(). */ - sleep(3); - NNTI_interrupt(&trans_hdl); - pthread_barrier_wait(&barrier); - - pthread_barrier_wait(&barrier); - NNTI_interrupt(&trans_hdl); - pthread_barrier_wait(&barrier); - /* between these barriers the wait thread is calling NNTI_waitall(). */ - pthread_barrier_wait(&barrier); - /* between these barriers the wait thread is calling NNTI_waitall(). */ - sleep(3); - NNTI_interrupt(&trans_hdl); - pthread_barrier_wait(&barrier); - - join_wait_thread(); - - NNTI_free(&mr1); - NNTI_free(&mr2); - NNTI_free(&mr3); - - log_debug(interrupt_debug_level, "Time to complete NNTI_wait: expected=%lums ; actual=%lums", (uint64_t)expected_wait_time, (uint64_t)actual_wait_time); - if ((actual_wait_time < (expected_wait_time-100)) || - (actual_wait_time > (expected_wait_time+100))) { - success=FALSE; - } - - NNTI_fini(&trans_hdl); - - logger_fini(); - - if (success) - fprintf(stdout, "\nEnd Result: TEST PASSED\n"); - else - fprintf(stdout, "\nEnd Result: TEST FAILED\n"); - - return (success ? 0 : 1 ); -} diff --git a/packages/trios/tests/unit/LoggerTest.cpp b/packages/trios/tests/unit/LoggerTest.cpp deleted file mode 100644 index 2821b5cc9794..000000000000 --- a/packages/trios/tests/unit/LoggerTest.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -#include -#include - - -#include "Trios_logger.h" - - -int main(int argc, char *argv[]) { - - log_level debug_level=LOG_ALL; - - logger_init(LOG_ERROR, NULL); - - log_debug(debug_level, "print number=%d", 10); - log_info(debug_level, "print number=%d", 11); - log_warn(debug_level, "warn msg"); - log_error(debug_level, "error msg"); - log_fatal(debug_level, "fatal msg"); - - logger_fini(); - - return 0; -} diff --git a/packages/trios/tests/unit/MultiUseBufferTest.cpp b/packages/trios/tests/unit/MultiUseBufferTest.cpp deleted file mode 100644 index 7e29a18b4e51..000000000000 --- a/packages/trios/tests/unit/MultiUseBufferTest.cpp +++ /dev/null @@ -1,849 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * NonContigTest.c - * - * Created on: March 14, 2014 - * Author: thkorde - */ - -#include "Trios_nnti.h" - -#include -#include - -#include - -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include - - -log_level multiuse_debug_level = LOG_UNDEFINED; - - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; - -NNTI_buffer_t queue_mr; -NNTI_work_request_t queue_wr; -NNTI_status_t queue_status; - -#define WR_COUNT 2 -NNTI_work_request_t *mr_wr_list[WR_COUNT]; -NNTI_result_t err; -NNTI_status_t wait_status1; -NNTI_status_t wait_status2; -NNTI_status_t wait_status3; -NNTI_status_t *wait_status_list[WR_COUNT]; - -int one_kb=1024; -int one_mb=1024*1024; - -#define NUM_SEGMENTS 10 - - -/* - * Can't use NNTI_REQUEST_BUFFER_SIZE or NNTI_RESULT_BUFFER_SIZE here, - * because the encoded NNTI buffers are too big. Define something - * bigger here. - */ -#define NNTI_MULTIUSE_REQUEST_SIZE 2048 -#define NNTI_MULTIUSE_RESULT_SIZE 2048 - - - -int client_test1(NNTI_buffer_t *local_multiuse_mr); -int client_test2(NNTI_buffer_t *local_multiuse_mr); -int client_test3(NNTI_buffer_t *local_multiuse_mr); -void server_test1(NNTI_buffer_t *local_multiuse_mr); -void server_test2(NNTI_buffer_t *local_multiuse_mr); -void server_test3(NNTI_buffer_t *local_multiuse_mr); - - - -static inline uint64_t calc_checksum (char * buf, uint64_t size) -{ - unsigned long hash = 5381; - char* p = buf; - uint64_t i = 0; - - for (i=0; i -#include -#include - -#include - -NNTI_transport_t trans_hdl, copy_trans_hdl; -NNTI_peer_t server_hdl, copy_server_hdl; -NNTI_buffer_t queue_mr, send_mr, copy_queue_mr, copy_send_mr; /* registered memory regions */ -NNTI_work_request_t queue_wr, send_wr, copy_queue_wr, copy_send_wr; /* active operations */ -NNTI_result_t err; -NNTI_status_t queue_status, send_status, copy_queue_status, copy_send_status; - -uint64_t packed_size; -char packed_buf[8096]; - - -pthread_barrier_t barrier; - -bool success=true; - -typedef struct { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -} data_t; - -typedef struct { - /* test data */ - data_t data; - /** 32-bit checksum of the test data */ - uint64_t chksum; -} selfsend_args; - -static uint64_t calc_checksum (const char * buf, const uint64_t size) -{ - unsigned long hash = 5381; - const char* p = buf; - uint64_t i = 0; - - for (i=0; ichksum != calc_checksum((const char *)&ssa->data, sizeof(data_t))) { - success=false; - } - - NNTI_dt_free(©_trans_hdl, ©_queue_status); - - pthread_barrier_wait(&barrier); - - NNTI_dt_free(©_trans_hdl, ©_queue_wr); - NNTI_destroy_work_request(&queue_wr); - - NNTI_dt_free(©_trans_hdl, ©_queue_mr); - NNTI_free(&queue_mr); - - return(NULL); -} - -static pthread_t wait_thread; -static void launch_wait_thread() -{ - /* Start up polling thread */ - fprintf (stdout, "Start wait thread.\n"); - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int rc = pthread_create(&wait_thread, &attr, do_wait, NULL); - if (rc) { - fprintf (stdout, "ERROR: Cannot create polling thread, err code = %d\n", rc); - return; - } - pthread_attr_destroy(&attr); -} -static void join_wait_thread() -{ - void *status; - int rc = pthread_join(wait_thread, &status); - if (rc) { - fprintf (stdout, "ERROR: Cannot join wait thread, err code = %d\n", rc); - } else { - fprintf (stdout, "Joined wait thread.\n"); - } -} - - -int main(int argc, char *argv[]) -{ - NNTI_result_t rc; - selfsend_args *ssa; - char server_url[NNTI_URL_LEN]; - - logger_init(LOG_ERROR, NULL); - - pthread_barrier_init(&barrier, NULL, 2); - - rc=NNTI_init(NNTI_DEFAULT_TRANSPORT, NULL, &trans_hdl); - - NNTI_dt_sizeof(&trans_hdl, &trans_hdl, &packed_size); - NNTI_dt_pack(&trans_hdl, &trans_hdl, packed_buf, packed_size); - NNTI_dt_unpack(&trans_hdl, ©_trans_hdl, packed_buf, packed_size); - - rc=NNTI_get_url(©_trans_hdl, server_url, NNTI_URL_LEN); - - launch_wait_thread(); - - pthread_barrier_wait(&barrier); - - rc=NNTI_connect( - ©_trans_hdl, - server_url, - 5000, - &server_hdl); - - NNTI_dt_sizeof(©_trans_hdl, &server_hdl, &packed_size); - NNTI_dt_pack(©_trans_hdl, &server_hdl, packed_buf, packed_size); - NNTI_dt_unpack(©_trans_hdl, ©_server_hdl, packed_buf, packed_size); - - rc=NNTI_alloc(©_trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, &send_mr); - - NNTI_dt_sizeof(©_trans_hdl, &send_mr, &packed_size); - NNTI_dt_pack(©_trans_hdl, &send_mr, packed_buf, packed_size); - NNTI_dt_unpack(©_trans_hdl, ©_send_mr, packed_buf, packed_size); - - ssa=(selfsend_args *)NNTI_BUFFER_C_POINTER(&send_mr); - ssa->data.int_val =10; - ssa->data.float_val =10.0; - ssa->data.double_val=10.0; - ssa->chksum=calc_checksum((const char *)&ssa->data, sizeof(data_t)); - - rc=NNTI_send(©_server_hdl, ©_send_mr, NULL, &send_wr); - - NNTI_dt_sizeof(©_trans_hdl, &send_wr, &packed_size); - NNTI_dt_pack(©_trans_hdl, &send_wr, packed_buf, packed_size); - NNTI_dt_unpack(©_trans_hdl, ©_send_wr, packed_buf, packed_size); - - rc=NNTI_wait(©_send_wr, 5000, &send_status); - - NNTI_dt_free(©_trans_hdl, ©_send_wr); - - pthread_barrier_wait(&barrier); - - NNTI_dt_free(©_trans_hdl, ©_send_mr); - NNTI_free(&send_mr); - - NNTI_dt_free(©_trans_hdl, ©_server_hdl); - rc=NNTI_disconnect( - ©_trans_hdl, - &server_hdl); - - join_wait_thread(); - - NNTI_dt_free(&trans_hdl, ©_trans_hdl); - NNTI_fini(&trans_hdl); - - if (success) - std::cout << "\nEnd Result: TEST PASSED" << std::endl; - else - std::cout << "\nEnd Result: TEST FAILED" << std::endl; - - return (success ? 0 : 1 ); -} diff --git a/packages/trios/tests/unit/NntiPerfTest.cpp b/packages/trios/tests/unit/NntiPerfTest.cpp deleted file mode 100644 index 2e6466bd74ea..000000000000 --- a/packages/trios/tests/unit/NntiPerfTest.cpp +++ /dev/null @@ -1,776 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * NntiPerfTest.c - * - * Created on: May 16, 2014 - * Author: thkorde - */ - -#include "Trios_nnti.h" -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nnti_fprint_types.h" - -#include "Teuchos_CommandLineProcessor.hpp" -#include "Teuchos_StandardCatchMacros.hpp" -#include "Teuchos_oblackholestream.hpp" - -#include -#include - -#include - - -int nprocs, nclients, nservers; -int rank, client_rank; - - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; -char url[NNTI_URL_LEN]; - -NNTI_buffer_t queue_mr; -NNTI_buffer_t client_ack_mr; -NNTI_buffer_t server_ack_mr; -NNTI_buffer_t send_mr; - -NNTI_work_request_t queue_wr; -NNTI_work_request_t client_ack_wr; -NNTI_work_request_t server_ack_wr; -NNTI_work_request_t *send_wr; // need an array of work requests because we will be issuing multiple async sends -NNTI_work_request_t **send_wr_list; - -NNTI_buffer_t get_src_mr; -NNTI_buffer_t get_dst_mr; -NNTI_buffer_t put_src_mr; -NNTI_buffer_t put_dst_mr; - -NNTI_work_request_t *get_wr; // need an array of work requests because we will be issuing multiple async gets -NNTI_work_request_t *put_wr; // need an array of work requests because we will be issuing multiple async puts - -NNTI_result_t err; - -int one_mb=1024*1024; - -log_level nntiperf_debug_level = LOG_UNDEFINED; - -uint64_t num_sends; - -uint64_t num_gets; -uint64_t min_get_size; -uint64_t max_get_size; - -uint64_t num_puts; -uint64_t min_put_size; -uint64_t max_put_size; - - -static int buffer_pack(void *input, char **output, uint64_t *output_size) -{ - NNTI_dt_sizeof(&trans_hdl, input, output_size); - *output=(char*)malloc(*output_size); - NNTI_dt_pack(&trans_hdl, input, *output, *output_size); - - return(0); -} - -static int buffer_free(void *input) -{ - NNTI_dt_free(&trans_hdl, input); - - return(0); -} - -static int buffer_unpack(char *input, uint64_t input_size, void *output) -{ - NNTI_dt_unpack(&trans_hdl, output, input, input_size); - - return(0); -} - -void client(void) { - NNTI_result_t rc=NNTI_OK; - NNTI_status_t rdma_status; - NNTI_status_t *send_status; - NNTI_status_t **send_status_list; - NNTI_status_t client_ack_status; - char *c_ptr; - char *packed=NULL; - uint64_t packed_size=0; - - double op_timer; - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 1 ? std::cout : blackhole ); - - NNTI_connect(&trans_hdl, url, 5000, &server_hdl); - - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, &send_mr); - char *send_buf=NNTI_BUFFER_C_POINTER(&send_mr); - memset(send_buf, 0, NNTI_REQUEST_BUFFER_SIZE); - - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_RECV_DST, &client_ack_mr); - char *client_ack_buf=NNTI_BUFFER_C_POINTER(&client_ack_mr); - memset(client_ack_buf, 0, NNTI_REQUEST_BUFFER_SIZE); - - NNTI_alloc(&trans_hdl, max_get_size, 1, NNTI_GET_DST, &get_dst_mr); - char *get_dst_buf=NNTI_BUFFER_C_POINTER(&get_dst_mr); - memset(get_dst_buf, 0, max_get_size); - - NNTI_alloc(&trans_hdl, max_put_size, 1, NNTI_PUT_SRC, &put_src_mr); - char *put_src_buf=NNTI_BUFFER_C_POINTER(&put_src_mr); - memset(put_src_buf, 0, max_put_size); - - send_wr = (NNTI_work_request_t*)malloc(num_sends*sizeof(NNTI_work_request_t)); - send_wr_list = (NNTI_work_request_t**)malloc(num_sends*sizeof(NNTI_work_request_t*)); - send_status = (NNTI_status_t*)malloc(num_sends*sizeof(NNTI_status_t)); - send_status_list = (NNTI_status_t**)malloc(num_sends*sizeof(NNTI_status_t*)); - for (int i=0;i NNTI_REQUEST_BUFFER_SIZE) { - log_error(nntiperf_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); - MPI_Abort(MPI_COMM_WORLD, -10); - } - - // send the server the recv_mr so it can send back it's ack_mr - memcpy(send_buf, packed, packed_size); - - NNTI_create_work_request(&client_ack_mr, &client_ack_wr); - - rc=NNTI_send(&server_hdl, &send_mr, NULL, &send_wr[0]); - if (rc != NNTI_OK) { - log_error(nntiperf_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_wr[0], 5000, &send_status[0]); - if (rc != NNTI_OK) { - log_error(nntiperf_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - - // wait for the server to send back it's recv_mr - rc=NNTI_wait(&client_ack_wr, -1, &client_ack_status); - - char *ptr=(char*)client_ack_status.start+client_ack_status.offset; - - memcpy(&packed_size, ptr, sizeof(packed_size)); - ptr += sizeof(packed_size); - memcpy(packed, ptr, packed_size); - ptr += packed_size; - - buffer_unpack(packed, packed_size, &server_ack_mr); - - memcpy(&packed_size, ptr, sizeof(packed_size)); - ptr += sizeof(packed_size); - memcpy(packed, ptr, packed_size); - ptr += packed_size; - - buffer_unpack(packed, packed_size, &get_src_mr); - - memcpy(&packed_size, ptr, sizeof(packed_size)); - ptr += sizeof(packed_size); - memcpy(packed, ptr, packed_size); - ptr += packed_size; - - buffer_unpack(packed, packed_size, &put_dst_mr); - - NNTI_destroy_work_request(&client_ack_wr); - -// fprint_NNTI_buffer(logger_get_file(), "server_ack_mr", -// "received server ack hdl", &server_ack_mr); -// fprint_NNTI_buffer(logger_get_file(), "get_src_mr", -// "received get src hdl", &get_src_mr); -// fprint_NNTI_buffer(logger_get_file(), "put_dst_mr", -// "received put dst hdl", &put_dst_mr); - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 2 - test sync request performance - */ - NNTI_create_work_request(&client_ack_mr, &client_ack_wr); - op_timer=trios_get_time(); - for (uint64_t i=0;i 0) { - out << " sync requests per second == " << num_sends/op_timer << std::endl; - out << " sync requests MBps == " << (double)(NNTI_REQUEST_BUFFER_SIZE * num_sends)/one_mb/op_timer << std::endl; - } - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 3 - test async request performance - */ - NNTI_create_work_request(&client_ack_mr, &client_ack_wr); - op_timer=trios_get_time(); - for (uint64_t i=0;i 0) { - out << "async requests per second == " << num_sends/op_timer << std::endl; - out << "async requests MBps == " << (double)(NNTI_REQUEST_BUFFER_SIZE * num_sends)/one_mb/op_timer << std::endl; - } - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 4 - test sync get performance - */ - // warm up the pipes - for (uint64_t i=0;i 0) { - out << " sync get (" << get_size << " byte transfer) == " << (double)(num_gets*get_size)/one_mb/op_timer << " MBps" << std::endl; - } - } - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 5 - test async get performance - */ - // warm up the pipes - for (uint64_t i=0;i 0) { - out << "async get (" << get_size << " byte transfer) == " << (double)(num_gets*get_size)/one_mb/op_timer << " MBps" << std::endl; - } - } - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 6 - test sync put performance - */ - // warm up the pipes - for (uint64_t i=0;i 0) { - out << " sync put (" << put_size << " byte transfer) == " << (double)(num_puts*put_size)/one_mb/op_timer << " MBps" << std::endl; - } - } - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 7 - test async put performance - */ - // warm up the pipes - for (uint64_t i=0;i 0) { - out << "async put (" << put_size << " byte transfer) == " << (double)(num_puts*put_size)/one_mb/op_timer << " MBps" << std::endl; - } - } - - MPI_Barrier(MPI_COMM_WORLD); - - free(send_wr); - free(get_wr); - free(put_wr); - - buffer_free(&server_ack_mr); - buffer_free(&get_src_mr); - buffer_free(&put_dst_mr); - - NNTI_free(&send_mr); - NNTI_free(&client_ack_mr); - NNTI_free(&get_dst_mr); - NNTI_free(&put_src_mr); - - return; -} - -void server(void) -{ - NNTI_result_t rc=NNTI_OK; - NNTI_status_t queue_status; - NNTI_status_t send_status; - NNTI_status_t server_ack_status; - char *c_ptr; - char *packed=NULL; - uint64_t packed_size=0; - - - int num_elements=nclients+(4*nclients*num_sends); - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, num_elements, NNTI_RECV_QUEUE, &queue_mr); - - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, &send_mr); - char *send_buf=NNTI_BUFFER_C_POINTER(&send_mr); - memset(send_buf, 0, NNTI_REQUEST_BUFFER_SIZE); - - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_RECV_DST, &server_ack_mr); - char *server_ack_buf=NNTI_BUFFER_C_POINTER(&server_ack_mr); - memset(server_ack_buf, 0, NNTI_REQUEST_BUFFER_SIZE); - - char *get_src_buf=(char *)malloc(nclients*max_get_size); - memset(get_src_buf, 0, nclients*max_get_size); - NNTI_register_memory(&trans_hdl, get_src_buf, nclients*max_get_size, 1, NNTI_GET_SRC, &get_src_mr); - - char *put_dst_buf=(char *)malloc(nclients*max_put_size); - memset(put_dst_buf, 0, nclients*max_put_size); - NNTI_register_memory(&trans_hdl, put_dst_buf, nclients*max_put_size, 1, NNTI_PUT_DST, &put_dst_mr); - - send_wr=(NNTI_work_request_t*)malloc(num_sends*sizeof(NNTI_work_request_t)); - - /* - * Phase 1 - exchange buffers handles - */ - // wait for the client to send it's recv_mr - NNTI_create_work_request(&queue_mr, &queue_wr); - - NNTI_wait(&queue_wr, -1, &queue_status); - - c_ptr=(char*)queue_status.start+queue_status.offset; - buffer_unpack(c_ptr, queue_status.length, &client_ack_mr); - - NNTI_destroy_work_request(&queue_wr); - -// fprint_NNTI_buffer(logger_get_file(), "client_ack_mr", -// "received client ack hdl", &client_ack_mr); - - // send our server_ack_mr, get_src_mr and put_dst_mr back to the client - buffer_pack(&server_ack_mr, &packed, &packed_size); - if (packed_size > NNTI_REQUEST_BUFFER_SIZE) { - log_error(nntiperf_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); - MPI_Abort(MPI_COMM_WORLD, -10); - } - - char *ptr=send_buf; - memcpy(ptr, &packed_size, sizeof(packed_size)); - ptr += sizeof(packed_size); - memcpy(ptr, packed, packed_size); - ptr += packed_size; - - free(packed); - - buffer_pack(&get_src_mr, &packed, &packed_size); - if (packed_size > NNTI_REQUEST_BUFFER_SIZE) { - log_error(nntiperf_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); - MPI_Abort(MPI_COMM_WORLD, -10); - } - - memcpy(ptr, &packed_size, sizeof(packed_size)); - ptr += sizeof(packed_size); - memcpy(ptr, packed, packed_size); - ptr += packed_size; - - free(packed); - - buffer_pack(&put_dst_mr, &packed, &packed_size); - if (packed_size > NNTI_REQUEST_BUFFER_SIZE) { - log_error(nntiperf_debug_level, "buffer_pack() says encoded NNTI_buffer_t is larger than NNTI_REQUEST_BUFFER_SIZE"); - MPI_Abort(MPI_COMM_WORLD, -10); - } - - memcpy(ptr, &packed_size, sizeof(packed_size)); - ptr += sizeof(packed_size); - memcpy(ptr, packed, packed_size); - ptr += packed_size; - - free(packed); - - rc=NNTI_send(&queue_status.src, &send_mr, &client_ack_mr, &send_wr[0]); - if (rc != NNTI_OK) { - log_error(nntiperf_debug_level, "NNTI_send() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - rc=NNTI_wait(&send_wr[0], 5000, &send_status); - if (rc != NNTI_OK) { - log_error(nntiperf_debug_level, "NNTI_wait() returned an error: %d", rc); - MPI_Abort(MPI_COMM_WORLD, rc); - } - - MPI_Barrier(MPI_COMM_WORLD); - - /* - * Phase 2 - client sends sync requests - */ - for (uint64_t i=0;i \n", argv[0]); - MPI_Barrier(MPI_COMM_WORLD); - } else { - MPI_Barrier(MPI_COMM_WORLD); - } - MPI_Abort(MPI_COMM_WORLD, -1); - } - - env_debug_level=getenv("NNTI_LOG_LEVEL"); - if (env_debug_level != NULL) { - debug_level=(log_level)strtol(env_debug_level, NULL, 0); - sprintf(logname, "nntiperf.%03d.log", rank); - debug_name=&logname[0]; - } - logger_init(debug_level, debug_name); - - NNTI_init(NNTI_DEFAULT_TRANSPORT, NULL, &trans_hdl); - - if (rank==0) { - NNTI_get_url(&trans_hdl, url, NNTI_URL_LEN); - } - - MPI_Bcast(&url[0], NNTI_URL_LEN, MPI_CHAR, 0, MPI_COMM_WORLD); - - log_debug(nntiperf_debug_level, "NNTI perfermance server url is %s", url); - - if (rank==0) { - server(); - } else { - client(); - } - - NNTI_fini(&trans_hdl); - - MPI_Finalize(); - - logger_fini(); - - Teuchos::oblackholestream blackhole; - std::ostream &out = ( rank == 1 ? std::cout : blackhole ); - if (success) - out << "\nEnd Result: TEST PASSED" << std::endl; - else - out << "\nEnd Result: TEST FAILED" << std::endl; - - return (success ? 0 : 1 ); -// return 0; -} diff --git a/packages/trios/tests/unit/NntiSelfSendTest.cpp b/packages/trios/tests/unit/NntiSelfSendTest.cpp deleted file mode 100644 index bd20cd63e87c..000000000000 --- a/packages/trios/tests/unit/NntiSelfSendTest.cpp +++ /dev/null @@ -1,210 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * SelfSendTest.c - * - * Created on: March 20, 2014 - * Author: thkorde - */ - -#include "Trios_nnti.h" - -#include "Trios_logger.h" -#include "Trios_timer.h" - -#include -#include -#include - -#include - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; -NNTI_buffer_t queue_mr, send_mr; /* registered memory regions */ -NNTI_work_request_t queue_wr, send_wr; /* registered memory regions */ -NNTI_result_t err; -NNTI_status_t queue_status, send_status; - -pthread_barrier_t barrier; - -bool success=true; - -typedef struct { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -} data_t; - -typedef struct { - /* test data */ - data_t data; - /** 32-bit checksum of the test data */ - uint64_t chksum; -} selfsend_args; - -static uint64_t calc_checksum (const char * buf, const uint64_t size) -{ - unsigned long hash = 5381; - const char* p = buf; - uint64_t i = 0; - - for (i=0; ichksum != calc_checksum((const char *)&ssa->data, sizeof(data_t))) { - success=false; - } - - pthread_barrier_wait(&barrier); - - NNTI_destroy_work_request(&queue_wr); - - NNTI_free(&queue_mr); - - return(NULL); -} - -static pthread_t wait_thread; -static void launch_wait_thread() -{ - /* Start up polling thread */ - fprintf (stdout, "Start wait thread.\n"); - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int rc = pthread_create(&wait_thread, &attr, do_wait, NULL); - if (rc) { - fprintf (stdout, "ERROR: Cannot create polling thread, err code = %d\n", rc); - return; - } - pthread_attr_destroy(&attr); -} -static void join_wait_thread() -{ - void *status; - int rc = pthread_join(wait_thread, &status); - if (rc) { - fprintf (stdout, "ERROR: Cannot join wait thread, err code = %d\n", rc); - } else { - fprintf (stdout, "Joined wait thread.\n"); - } -} - - -int main(int argc, char *argv[]) -{ - NNTI_result_t rc; - selfsend_args *ssa; - char server_url[NNTI_URL_LEN]; - - logger_init(LOG_ERROR, NULL); - - pthread_barrier_init(&barrier, NULL, 2); - - rc=NNTI_init(NNTI_DEFAULT_TRANSPORT, NULL, &trans_hdl); - rc=NNTI_get_url(&trans_hdl, server_url, NNTI_URL_LEN); - - launch_wait_thread(); - - pthread_barrier_wait(&barrier); - - rc=NNTI_connect( - &trans_hdl, - server_url, - 5000, - &server_hdl); - - rc=NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, &send_mr); - - ssa=(selfsend_args *)NNTI_BUFFER_C_POINTER(&send_mr); - ssa->data.int_val =10; - ssa->data.float_val =10.0; - ssa->data.double_val=10.0; - ssa->chksum=calc_checksum((const char *)&ssa->data, sizeof(data_t)); - - rc=NNTI_send(&server_hdl, &send_mr, NULL, &send_wr); - rc=NNTI_wait(&send_wr, 5000, &send_status); - - pthread_barrier_wait(&barrier); - - NNTI_free(&send_mr); - - join_wait_thread(); - - NNTI_fini(&trans_hdl); - - if (success) - std::cout << "\nEnd Result: TEST PASSED" << std::endl; - else - std::cout << "\nEnd Result: TEST FAILED" << std::endl; - - return (success ? 0 : 1 ); -} diff --git a/packages/trios/tests/unit/NonContigTest.cpp b/packages/trios/tests/unit/NonContigTest.cpp deleted file mode 100644 index 045714b8432d..000000000000 --- a/packages/trios/tests/unit/NonContigTest.cpp +++ /dev/null @@ -1,1547 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * NonContigTest.c - * - * Created on: March 14, 2014 - * Author: thkorde - */ - -#include "Trios_nnti.h" - -#include -#include - -#include - -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_nnti_fprint_types.h" - -#include -#include - - -log_level noncontig_debug_level = LOG_UNDEFINED; - - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; - -NNTI_buffer_t contig_queue_mr; -NNTI_work_request_t contig_queue_wr; -NNTI_status_t queue_status; - -/* - * all tests - */ -NNTI_buffer_t contig_send_mr, contig_recv_mr; /* contiguous send/recv registered memory regions */ -NNTI_work_request_t contig_send_wr, contig_recv_wr; /* work requests */ - -#define WR_COUNT 2 -NNTI_work_request_t *mr_wr_list[WR_COUNT]; -NNTI_result_t err; -NNTI_status_t wait_status1; -NNTI_status_t wait_status2; -NNTI_status_t wait_status3; -NNTI_status_t *wait_status_list[WR_COUNT]; - -int one_kb=1024; -int one_mb=1024*1024; - -#define NUM_SEGMENTS 5 - - -/* - * Can't use NNTI_REQUEST_BUFFER_SIZE or NNTI_RESULT_BUFFER_SIZE here, - * because the encoded NNTI buffers are too big. Define something - * bigger here. - */ -#define NNTI_NONCONTIG_REQUEST_SIZE 2048 -#define NNTI_NONCONTIG_RESULT_SIZE 2048 - - -static inline uint64_t calc_checksum (char * buf, uint64_t size) -{ - unsigned long hash = 5381; - char* p = buf; - uint64_t i = 0; - - for (i=0; i -#include - -#include - -log_level selfsend_debug_level = LOG_UNDEFINED; - -std::string my_url(NSSI_URL_LEN, '\0'); - -pthread_barrier_t barrier; - -bool success=true; - -static void *do_wait(void *args) -{ - nssi_service svc; - - // service init - nssi_service_init(NSSI_DEFAULT_TRANSPORT, NSSI_SHORT_REQUEST_SIZE, &svc); - nssi_get_url(NSSI_DEFAULT_TRANSPORT, &my_url[0], NSSI_URL_LEN); - - // client is waiting for us to initialize - pthread_barrier_wait(&barrier); - - svc.max_reqs = -1; - int rc = nssi_service_start(&svc); - if (rc != NSSI_OK) { - log_info(selfsend_debug_level, "exited selfsend_svc: %s", - nssi_err_str(rc)); - } - - /* finalize the service */ - nssi_service_fini(&svc); - - return(NULL); -} - -static pthread_t wait_thread; -static void launch_wait_thread() -{ - /* Start up polling thread */ - fprintf (stdout, "Start wait thread.\n"); - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int rc = pthread_create(&wait_thread, &attr, do_wait, NULL); - if (rc) { - fprintf (stdout, "ERROR: Cannot create polling thread, err code = %d\n", rc); - return; - } - pthread_attr_destroy(&attr); -} -static void join_wait_thread() -{ - void *status; - int rc = pthread_join(wait_thread, &status); - if (rc) { - fprintf (stdout, "ERROR: Cannot join wait thread, err code = %d\n", rc); - } else { - fprintf (stdout, "Joined wait thread.\n"); - } -} - - -int main(int argc, char *argv[]) -{ - int rc; - nssi_service svc; - - logger_init(LOG_ERROR, NULL); - - pthread_barrier_init(&barrier, NULL, 2); - - // common init - nssi_rpc_init(NSSI_DEFAULT_TRANSPORT, NSSI_DEFAULT_ENCODE, NULL); - - launch_wait_thread(); - - // wait for the service to initialize - pthread_barrier_wait(&barrier); - - // client init - nssi_init(NSSI_DEFAULT_TRANSPORT); - - for (int i=0; i < 3; i++) { - log_debug(selfsend_debug_level, "Try to connect to server: attempt #%d", i); - rc=nssi_get_service(NSSI_DEFAULT_TRANSPORT, my_url.c_str(), 5000, &svc); - if (rc == NSSI_OK) - break; - else if (rc != NSSI_ETIMEDOUT) { - log_error(selfsend_debug_level, "could not get svc description: %s", - nssi_err_str(rc)); - break; - } - } - - sleep(1); - - // shutdown the service - rc = nssi_kill(&svc, 0, 5000); - if (rc != NSSI_OK) { - log_error(selfsend_debug_level, "Error in nssi_kill"); - goto err; - } - - // finalize the client - nssi_fini(NSSI_DEFAULT_TRANSPORT); - - join_wait_thread(); - - // finalize the NSSI library - rc = nssi_rpc_fini(NSSI_DEFAULT_TRANSPORT); - if (rc != NSSI_OK) { - log_error(selfsend_debug_level, "Error in nssi_rpc_fini"); - goto err; - } - - logger_fini(); - - std::cout << "\nEnd Result: TEST PASSED" << std::endl; - - return 0; - -err: - std::cout << "\nEnd Result: TEST FAILED" << std::endl; - - return 1; -} diff --git a/packages/trios/tests/unit/NssiWaitTest.cpp b/packages/trios/tests/unit/NssiWaitTest.cpp deleted file mode 100644 index cc0c643bc6c1..000000000000 --- a/packages/trios/tests/unit/NssiWaitTest.cpp +++ /dev/null @@ -1,337 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * NssiWaitTest.c - * - * Created on: August 14, 2014 - * Author: thkorde - */ - -#include "Trios_nssi_client.h" -#include "Trios_nssi_server.h" -#include "Trios_nssi_fprint_types.h" - -#include -#include - -#include - -#include - -#include "Trios_logger.h" -#include "Trios_timer.h" -#include "Trios_trace.h" - -#include "nssi_opcodes.h" -#include "nssi_service_args.h" - -#include -#include - - -log_level nssiwait_debug_level = LOG_UNDEFINED; - - -#define REQ_COUNT 5 - - -int wait_test(const nssi_service *svc) -{ - static const char *empty=""; - - int rc = NSSI_OK; - int rc2 = NSSI_OK; - - int timeout=5000; - nssi_request req[REQ_COUNT]; - - nssi_trace_reset_args args; - - memset(&args, 0, sizeof(nssi_trace_reset_args)); - - /* initialize the args */ - args.fname = (char *)empty; - args.ftype = 0; - args.enable = (char *)empty; - - log_debug(nssiwait_debug_level, "Calling nssi_trace_reset(%s, %d, %s)\n", - args.fname, args.ftype, args.enable); - - /* send the request */ - for (int i=0;i -#include -#include - -#include - -NNTI_transport_t trans_hdl; -NNTI_peer_t server_hdl; -NNTI_buffer_t queue_mr, recv_mr, send_mr; /* registered memory regions */ -NNTI_work_request_t queue_wr, recv_wr, send_wr; /* registered memory regions */ -NNTI_result_t err; -NNTI_status_t queue_status, recv_status, send_status; - -pthread_barrier_t barrier2; -pthread_barrier_t barrier3; - -bool success=true; - -typedef struct { - /** An integer value. */ - uint32_t int_val; - /** A floating point value. */ - float float_val; - /** A double value. */ - double double_val; -} data_t; - -typedef struct { - /* test data */ - data_t data; - /** 32-bit checksum of the test data */ - uint64_t chksum; -} selfsend_args; - -static uint64_t calc_checksum (const char * buf, const uint64_t size) -{ - unsigned long hash = 5381; - const char* p = buf; - uint64_t i = 0; - - for (i=0; ichksum != calc_checksum((const char *)&ssa->data, sizeof(data_t))) { - fprintf(stdout, "checksum failure in the request thread"); - success=false; - } - - pthread_barrier_wait(&barrier3); - - NNTI_destroy_work_request(&queue_wr); - - NNTI_free(&queue_mr); - - return(NULL); -} - -#define RECV_TIMEOUT 3000 -static void *do_recv_wait(void *args) -{ - NNTI_result_t rc; - selfsend_args *ssa; - - double wait_time=0.0; - - NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_RECV_DST, &recv_mr); - - NNTI_create_work_request(&recv_mr, &recv_wr); - - /* client is waiting for us to initialize */ - pthread_barrier_wait(&barrier3); - - wait_time=trios_get_time_ms(); - /* the client sends a message here */ - NNTI_wait(&recv_wr, RECV_TIMEOUT, &recv_status); - wait_time=trios_get_time_ms()-wait_time; - - // if wait time varies from the timeout by more than 100ms, then fail. - if ((wait_time < (RECV_TIMEOUT-100)) || (wait_time > (RECV_TIMEOUT+100))) { - fprintf(stdout, "Time to complete NNTI_wait: expected=%lums ; actual=%lums\n", (uint64_t)RECV_TIMEOUT, (uint64_t)wait_time); - success=false; - } - - pthread_barrier_wait(&barrier2); - - wait_time=trios_get_time_ms(); - /* the client sends a message here */ - NNTI_wait(&recv_wr, -1, &recv_status); - wait_time=trios_get_time_ms()-wait_time; - - ssa=(selfsend_args *)recv_status.start; - if (ssa->chksum != calc_checksum((const char *)&ssa->data, sizeof(data_t))) { - fprintf(stdout, "checksum failure in the receive thread"); - success=false; - } - - pthread_barrier_wait(&barrier3); - - NNTI_destroy_work_request(&recv_wr); - - NNTI_free(&recv_mr); - - return(NULL); -} - -static pthread_t request_thread; -static pthread_t recv_thread; -static void launch_wait_threads() -{ - int rc=0; - - /* Start up polling threads */ - fprintf (stdout, "Start wait threads.\n"); - pthread_attr_t attr; - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - rc = pthread_create(&request_thread, &attr, do_request_wait, NULL); - if (rc) { - fprintf (stdout, "ERROR: Cannot create request thread, err code = %d\n", rc); - return; - } - rc = pthread_create(&recv_thread, &attr, do_recv_wait, NULL); - if (rc) { - fprintf (stdout, "ERROR: Cannot create receive thread, err code = %d\n", rc); - return; - } - pthread_attr_destroy(&attr); -} -static void join_wait_threads() -{ - int rc=0; - - void *status; - rc = pthread_join(request_thread, &status); - if (rc) { - fprintf (stdout, "ERROR: Cannot join request thread, err code = %d\n", rc); - } else { - fprintf (stdout, "Joined request thread.\n"); - } - rc = pthread_join(recv_thread, &status); - if (rc) { - fprintf (stdout, "ERROR: Cannot join receive thread, err code = %d\n", rc); - } else { - fprintf (stdout, "Joined receive thread.\n"); - } -} - - -int main(int argc, char *argv[]) -{ - NNTI_result_t rc; - selfsend_args *ssa; - char server_url[NNTI_URL_LEN]; - - logger_init(LOG_ERROR, NULL); - - pthread_barrier_init(&barrier2, NULL, 2); - pthread_barrier_init(&barrier3, NULL, 3); - - rc=NNTI_init(NNTI_DEFAULT_TRANSPORT, NULL, &trans_hdl); - rc=NNTI_get_url(&trans_hdl, server_url, NNTI_URL_LEN); - - launch_wait_threads(); - - pthread_barrier_wait(&barrier3); - - rc=NNTI_connect( - &trans_hdl, - server_url, - 5000, - &server_hdl); - - pthread_barrier_wait(&barrier2); - - rc=NNTI_alloc(&trans_hdl, NNTI_REQUEST_BUFFER_SIZE, 1, NNTI_SEND_SRC, &send_mr); - - ssa=(selfsend_args *)NNTI_BUFFER_C_POINTER(&send_mr); - ssa->data.int_val =10; - ssa->data.float_val =10.0; - ssa->data.double_val=10.0; - ssa->chksum=calc_checksum((const char *)&ssa->data, sizeof(data_t)); - - rc=NNTI_send(&server_hdl, &send_mr, &recv_mr, &send_wr); - rc=NNTI_wait(&send_wr, 5000, &send_status); - - rc=NNTI_send(&server_hdl, &send_mr, NULL, &send_wr); - rc=NNTI_wait(&send_wr, 5000, &send_status); - - pthread_barrier_wait(&barrier3); - - NNTI_free(&send_mr); - - join_wait_threads(); - - if (success) - std::cout << "\nEnd Result: TEST PASSED" << std::endl; - else - std::cout << "\nEnd Result: TEST FAILED" << std::endl; - - return (success ? 0 : 1 ); -} diff --git a/packages/trios/tests/unit/SignalTest.c b/packages/trios/tests/unit/SignalTest.c deleted file mode 100644 index 52713d4959ef..000000000000 --- a/packages/trios/tests/unit/SignalTest.c +++ /dev/null @@ -1,56 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ -/* - * signal_test.c - * - * Created on: Aug 24, 2008 - * Author: raoldfi - */ - -#include "Trios_signal.h" - -int main(int argc, char *argv[]) -{ - trios_install_sighandler(); - return 0; -} diff --git a/packages/trios/tests/unit/TimerTest.c b/packages/trios/tests/unit/TimerTest.c deleted file mode 100644 index 751728284885..000000000000 --- a/packages/trios/tests/unit/TimerTest.c +++ /dev/null @@ -1,98 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include -#include -#include "Trios_timer.h" - -int main(int argc, char *argv[]) -{ - int sleeptime = 3; - - - uint64_t start_ns=0, end_ns=0; - uint64_t start_us, end_us; - uint64_t start_ms, end_ms; - double start_sec, end_sec; - - printf("Test trios_timer (This is not a resolution test)\n"); - - /* output the timer implementation we are using */ - printf("Timer implementation = %s\n\n", trios_timer_getimpl()); - - trios_get_time_ns(); - sleep(1); - - - start_ns = trios_get_time_ns(); - start_us = trios_get_time_us(); - start_ms = trios_get_time_ms(); - start_sec = trios_get_time(); - - /* sleep three seconds */ - sleep(sleeptime); - - end_ns = trios_get_time_ns(); - end_us = trios_get_time_us(); - end_ms = trios_get_time_ms(); - end_sec = trios_get_time(); - - printf("slept for %d seconds:\n", sleeptime); - printf("\tns = %lu\n", (unsigned long)(end_ns-start_ns)); - printf("\tus = %lu\n", (unsigned long)(end_us-start_us)); - printf("\tms = %lu\n", (unsigned long)(end_ms-start_ms)); - printf("\tsec = %f\n", end_sec-start_sec); - - - fprintf(stdout, "\nCalling trios_timer_test() ... "); - - if (trios_timer_test() != 0) { - fprintf(stdout, "failed\n"); - return -1; - } - else { - fprintf(stdout, "passed\n"); - } - - return 0; -} diff --git a/packages/trios/tests/unit/TraceTest.c b/packages/trios/tests/unit/TraceTest.c deleted file mode 100644 index fd98a974bff1..000000000000 --- a/packages/trios/tests/unit/TraceTest.c +++ /dev/null @@ -1,108 +0,0 @@ -/** -//@HEADER -// ************************************************************************ -// -// Trios: Trilinos I/O Support -// Copyright 2011 Sandia Corporation -// -// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, -// the U.S. Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the Corporation nor the names of the -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE -// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -//Questions? Contact Ron A. Oldfield (raoldfi@sandia.gov) -// -// ************************************************************************* -//@HEADER - */ - -#include -#include "Trios_trace.h" - -static int gid; - -static void testFunction( int n ) -{ - int j; - int pid = 0; - - trace_event(gid, 3, pid, "entered testFunction"); - - for (j = 0; j < n; j++) { - trace_event(gid, 2, pid, "inside for(j) loop"); - } - - trace_event(gid, 4, pid, "exiting testFunction"); -} - -int main(int argc, char *argv[]) -{ - int i; - int end = 3; - int event_id = 0; - int subevent_id = 1; - int thread_id = 0; - char fname[256]; - - - /* assign a file name for the trace file */ - sprintf(fname, "%s.sddf", argv[0]); - trace_init(fname, TRACE_SDDF_BINARY); - - - /* register a trace group */ - trace_register_group("test", &gid); - - - /* turn on tracing for the test group */ - trace_enable_gid(gid); - - /* write a generic trace event */ - trace_event(gid, event_id, thread_id, "start"); - - trace_start_interval(gid, thread_id); - - for (i = 1; i <= end; i++) { - trace_start_interval(gid, thread_id); - trace_inc_count(gid, 2, thread_id, "loop counter"); - - trace_event(gid, 1, thread_id, "top of for(i) loop"); - testFunction(i); - /*sleep(end-i);*/ - sleep(1); - trace_end_interval(gid, subevent_id, thread_id, "internal loop interval"); - } - - trace_end_interval(gid, event_id, thread_id, "outside loop interval"); - - trace_put_all_counts(gid, thread_id, "put all counts"); - - trace_fini(); - return 0; -} - diff --git a/packages/trios/tests/unit/cmake/Dependencies.cmake b/packages/trios/tests/unit/cmake/Dependencies.cmake deleted file mode 100644 index 92ab22a93dbd..000000000000 --- a/packages/trios/tests/unit/cmake/Dependencies.cmake +++ /dev/null @@ -1,8 +0,0 @@ -SET(LIB_REQUIRED_DEP_PACKAGES Triosnssi) -SET(LIB_OPTIONAL_DEP_PACKAGES) -SET(TEST_REQUIRED_DEP_PACKAGES) -SET(TEST_OPTIONAL_DEP_PACKAGES) -SET(LIB_REQUIRED_DEP_TPLS) -SET(LIB_OPTIONAL_DEP_TPLS) -SET(TEST_REQUIRED_DEP_TPLS) -SET(TEST_OPTIONAL_DEP_TPLS)