feature/gemv rps test -- RAJAPerf Suite Version of the BLAS2 GEMV Test

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tpls/rajaperf"]
+	path = tpls/rajaperf
+	url = https://github.com/ajpowelsnl/RAJAPerf.git

CMakeLists.txt

@@ -63,8 +63,20 @@ IF (NOT KOKKOSKERNELS_HAS_TRILINOS)
     BOOL
     "Whether to build tests. Default: OFF"
   )
+  KOKKOSKERNELS_ADD_OPTION(
+          "ENABLE_TESTS_AND_PERFSUITE"
+          OFF
+          BOOL
+          "Whether to build tests including Perfsuite. Default: OFF"
+  )
 ENDIF()
-
+IF(KokkosKernels_ENABLE_TESTS_AND_PERFSUITE)
+  set(BLT_CODE_CHECK_TARGET_NAME "fix-for-blt" CACHE STRING "Docstring")
+  set(INFRASTRUCTURE_ONLY ON CACHE BOOL "Only build the RAJAPerf infrastructure, no builtin kernels")
+  add_definitions("-DRAJAPERF_INFRASTRUCTURE_ONLY")
+  add_subdirectory(tpls/rajaperf)
+  include_directories(tpls/rajaperf/src)
+endif()
 SET(KokkosKernels_INSTALL_TESTING OFF CACHE INTERNAL
     "Whether to build tests and examples against installation")
 IF (KokkosKernels_INSTALL_TESTING)

cm_generate_makefile.bash

@@ -214,6 +214,8 @@ get_kernels_extra_linker_flags() {
   KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD="-DCMAKE_EXE_LINKER_FLAGS=${PARSE_EXTRA_LINKER_FLAGS_LIST} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD}"
 }
 
+
+
 display_help_text() {
 
       echo "KokkosKernels and Kokkos configure options:"
@@ -331,6 +333,10 @@ display_help_text() {
       echo "--gcc-toolchain=/Path/To/GccRoot:  Set the gcc toolchain to use with clang (e.g. /usr)" 
       echo "--kokkos-make-j=[NUM]:        Set -j parallel level for kokkos install"
       echo "                                Default: j == 4"
+      echo "--enable-tests: build Kokkos Kernels unit and performance tests"
+      echo "--enable-perfsuite: build Kokkos Kernels performance tests with
+RAJAPerf Suite"
+
 
 }
 
@@ -339,6 +345,7 @@ KOKKOS_INSTALL_PATH=""
 KOKKOS_DO_TESTS=OFF
 KOKKOS_DO_EXAMPLES=OFF
 KOKKOSKERNELS_DO_TESTS=ON
+KOKKOSKERNELS_DO_PERFSUITE=OFF
 KOKKOSKERNELS_DO_EXAMPLES=ON
 
 KOKKOS_MAKEINSTALL_J=4
@@ -491,6 +498,13 @@ do
       # This is the default
       KOKKOSKERNELS_DO_TESTS=ON
       ;;
+    --enable-perfsuite)
+      KOKKOSKERNELS_DO_PERFSUITE=ON
+      ;;
+    --disable-perfsuite)
+      # This is the default
+      KOKKOSKERNELS_DO_PERFSUITE=OFF
+      ;;
     --disable-tests)
       KOKKOSKERNELS_DO_TESTS=OFF
       ;;
@@ -744,7 +758,7 @@ cd $STORE_KOKKOSKERNELS_BUILD_PATH
 
 # Configure kokkos-kernels
 echo ""
-echo cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
+echo cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS_AND_PERFSUITE=${KOKKOSKERNELS_DO_PERFSUITE} -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
 echo ""
-cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
+cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS_AND_PERFSUITE=${KOKKOSKERNELS_DO_PERFSUITE} -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
 

cmake/KokkosKernels_config.h.in

@@ -22,6 +22,9 @@
    than just BLAS and LAPACK functions.  */
 #cmakedefine HAVE_KOKKOSKERNELS_MKL
 
+
+#cmakedefine KOKKOSKERNELS_ENABLE_TESTS_AND_PERFSUITE
+
 /* Define this macro if experimental features of Kokkoskernels are enabled */
 #cmakedefine HAVE_KOKKOSKERNELS_EXPERIMENTAL
 

cmake/kokkoskernels_tribits.cmake

@@ -152,6 +152,11 @@ IF (IS_ENABLED)
       TESTONLYLIBS ${PARSE_TESTONLYLIBS})
   ELSE()
     ADD_EXECUTABLE(${EXE_NAME} ${PARSE_SOURCES})
+    #AJP, BMK altered:
+    IF(KOKKOSKERNELS_ENABLE_TESTS_AND_PERFSUITE)
+      TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE common ${PARSE_TESTONLYLIBS})
+    ENDIF()
+
     IF (PARSE_TESTONLYLIBS)
       TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE Kokkos::kokkoskernels ${PARSE_TESTONLYLIBS})
     ELSE ()

perf_test/CMakeLists.txt

@@ -23,6 +23,22 @@ TARGET_INCLUDE_DIRECTORIES(kokkoskernelsperf_gtest PUBLIC $<BUILD_INTERFACE:${GT
 
 #Gtest minimally requires C++11
 TARGET_COMPILE_FEATURES(kokkoskernelsperf_gtest PUBLIC cxx_std_11)
+include_directories(sparse)
+if(KokkosKernels_ENABLE_TESTS_AND_PERFSUITE)
+# Add RPS implementations of KK perf tests here
+
+KOKKOSKERNELS_ADD_EXECUTABLE(
+        tracked_testing
+        SOURCES KokkosKernelsTrackedTesting.cpp
+        sparse/KokkosSparse_spmv_test.cpp
+        blas/blas2/KokkosBlas2_gemv_perf_test_rps.cpp
+        blas/blas1/KokkosBlas_dot_perf_test_rps.cpp
+        blas/blas1/KokkosBlas_team_dot_perf_test_rps.cpp
+        PerfTestUtilities.cpp
+        sparse/spmv/OpenMPSmartStatic_SPMV.cpp
+        #sparse/KokkosSparse_spgemm_test.cpp
+)
+endif()
 
 
 
@@ -32,4 +48,3 @@ ADD_COMPONENT_SUBDIRECTORY(sparse)
 ADD_COMPONENT_SUBDIRECTORY(blas)
 ADD_SUBDIRECTORY(performance)
 #ADD_SUBDIRECTORY(common)
-

perf_test/KokkosDot_test.cpp

@@ -0,0 +1,128 @@
+/*
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 3.0
+//       Copyright (2020) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+//
+// 3. Neither the name of the Corporation nor the names of the
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
+// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact Siva Rajamanickam ([email protected])
+//
+// ************************************************************************
+//@HEADER
+*/
+#include <cstdio>
+
+#include <ctime>
+#include <cstring>
+#include <cstdlib>
+#include <limits>
+#include <limits.h>
+#include <cmath>
+#include <unordered_map>
+#include <Kokkos_Core.hpp>
+#include <KokkosSparse_CrsMatrix.hpp>
+#include <KokkosKernels_IOUtils.hpp>
+#include "KokkosKernels_default_types.hpp"
+#include <common/RunParams.hpp>
+#include <common/QuickKernelBase.hpp>
+#include <PerfTestUtilities.hpp>
+
+#ifdef HAVE_CUSPARSE
+#include <CuSparse_SPMV.hpp>
+#endif
+
+#ifdef HAVE_MKL
+#include <MKL_SPMV.hpp>
+#endif
+
+#ifdef KOKKOS_ENABLE_OPENMP
+#include <OpenMPStatic_SPMV.hpp>
+#include <OpenMPDynamic_SPMV.hpp>
+#include <OpenMPSmartStatic_SPMV.hpp>
+#endif
+
+///////////////////////////////////////////////////////////////
+// RAJAPerf Suite interface for Kokkos, Kokkos-Kernels
+#include <common/Executor.hpp>
+
+// Headers from dot test prog
+
+#include <iostream>
+#include "KokkosBlas1_dot.hpp"
+#include "KokkosKernels_Utils.hpp"
+// in test_common
+//#include "KokkosKernels_TestUtils.hpp"
+
+
+  //FUNCTION THAT IS PART OF KK for generating test matrices
+  //create_random_x_vector and create_random_y_vector can be used together to generate a random 
+  //linear system Ax = y.
+  template<typename vec_t>
+  vec_t create_random_x_vector(vec_t& kok_x, double max_value = 10.0) {
+    typedef typename vec_t::value_type scalar_t;
+    auto h_x = Kokkos::create_mirror_view (kok_x);
+    for (size_t j = 0; j < h_x.extent(1); ++j){
+      for (size_t i = 0; i < h_x.extent(0); ++i){
+        scalar_t r =
+            static_cast <scalar_t> (rand()) /
+            static_cast <scalar_t> (RAND_MAX / max_value);
+        h_x.access(i, j) = r;
+      }
+    }
+    Kokkos::deep_copy (kok_x, h_x);
+    return kok_x;
+  }
+
+
+DotTestData setup_test(DotTestData::matrix_type A_matrix, DotTestData::matrix_type B_matrix) {
+          DotTestData test_data;
+          using matrix_type   = DotTestData::matrix_type;
+          test_data.A_matrix = create_random_x_vector(A_matrix);
+          // rm line 113 if build is good
+          //test.B_matrix = create_random_x_vector(B_matrix);
+          test_data.B_matrix = create_random_x_vector(B_matrix);
+
+        return test_data;
+}
+
+
+
+void run_benchmark(DotTestData& test_data) {
+
+        Kokkos::Timer timer;
+        Kokkos::fence();
+        timer.reset();
+        double result_1D = KokkosBlas::dot(test_data.A_matrix, test_data.B_matrix);
+        double elapsed = timer.seconds();
+}

perf_test/KokkosKernelsTrackedTesting.cpp

@@ -0,0 +1,75 @@
+//
+// Created by Poliakoff, David Zoeller on 4/26/21.
+//
+#include <common/RAJAPerfSuite.hpp>
+#include <common/Executor.hpp>
+#include "sparse/tracked_testing.hpp"
+#include <iostream>
+#include <Kokkos_Core.hpp>
+// For RPS implementation of BLAS Level-1 Tests
+#include "blas/blas1/tracked_testing.hpp"
+
+#include "blas/blas2/tracked_testing.hpp"
+
+
+int main(int argc, char* argv[]) {
+  {
+
+    // argument parsing for setting input data at runtime
+
+ 	std::string inputDataPath;
+		if (argc == 1) {
+		//    print_help();
+			std::cout << "Please provide input data directory: --input-data /PATH/TO/KOKKOS-KERNELS/INPUT/DATA" << std::endl; 
+			return 0;
+		  }
+
+  for (int i = 0; i < argc; i++) {
+    // if((strcmp(argv[i],"-v")==0)) {numVecs=atoi(argv[++i]); continue;}
+    if ((strcmp(argv[i], "--input-data") == 0)) {
+      i++;
+
+      if (i == argc) {
+        std::cerr << "Must pass desired input data after '--input-data'";
+        exit(1);
+      }
+      inputDataPath = std::string(argv[i]);
+      continue;
+    }
+}
+
+
+     test::set_input_data_path(inputDataPath);
+
+    // set up Executor
+    rajaperf::Executor exec(0, argv);
+    //rajaperf::Executor exec(argc, argv);
+    rajaperf::RunParams run_params(0, argv);
+    // Initialize Kokkos
+    Kokkos::initialize(argc, argv);
+
+    Kokkos::print_configuration(std::cout);
+
+     // sparse , spmv
+    test::sparse::build_executor(exec, argc, argv, run_params);
+
+    // All BLAS tests (Dot, Team Dot)
+    test::blas::build_blas_executor(exec, argc, argv, run_params);
+
+    test::blas2::build_blas2_executor(exec, argc, argv, run_params);
+
+    exec.setupSuite();
+
+    // STEP 3: Report suite run summary
+    //         (enable users to catch errors before entire suite is run)
+    exec.reportRunSummary(std::cout);
+
+    // STEP 4: Execute suite
+    exec.runSuite();
+
+    // STEP 5: Generate suite execution reports
+    exec.outputRunData();
+  }
+  Kokkos::finalize();
+return 0;
+}

perf_test/PerfTestUtilities.cpp

@@ -0,0 +1,27 @@
+//
+// Created by Poliakoff, David Zoeller on 4/27/21.
+//
+
+#include "KokkosKernels_default_types.hpp"
+#include "KokkosKernels_config.h"
+#include "KokkosKernels_IOUtils.hpp"
+#include <common/RunParams.hpp>
+#include <common/QuickKernelBase.hpp>
+#include <common/KernelBase.hpp>
+#include <dirent.h>
+
+
+namespace test {
+
+ std::string inputDataPath;
+
+ void set_input_data_path(const std::string& path_to_data){
+   inputDataPath = path_to_data;	
+
+};
+ std::string get_input_data_path(){
+
+return inputDataPath;
+
+};
+}