feature/gemv rps test -- RAJAPerf Suite Version of the BLAS2 GEMV Test

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tpls/rajaperf"]
+	path = tpls/rajaperf
+	url = https://github.com/ajpowelsnl/RAJAPerf.git

CMakeLists.txt

@@ -63,6 +63,19 @@ IF (NOT KOKKOSKERNELS_HAS_TRILINOS)
           BOOL
           "Whether to build tests. Default: OFF"
   )
+  KOKKOSKERNELS_ADD_OPTION(
+          "ENABLE_TESTS_AND_PERFSUITE"
+          OFF
+          BOOL
+          "Whether to build tests including Perfsuite. Default: OFF"
+  )
+  IF(KokkosKernels_ENABLE_TESTS_AND_PERFSUITE)
+    set(BLT_CODE_CHECK_TARGET_NAME "fix-for-blt" CACHE STRING "Docstring")
+    set(INFRASTRUCTURE_ONLY ON CACHE BOOL "Only build the RAJAPerf infrastructure, no builtin kernels")
+    add_definitions("-DRAJAPERF_INFRASTRUCTURE_ONLY")
+    add_subdirectory(tpls/rajaperf)
+    include_directories(tpls/rajaperf/src)
+  ENDIF()
 ENDIF ()
 
 KOKKOSKERNELS_ADD_OPTION(

cm_generate_makefile.bash

@@ -214,6 +214,8 @@ get_kernels_extra_linker_flags() {
   KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD="-DCMAKE_EXE_LINKER_FLAGS=${PARSE_EXTRA_LINKER_FLAGS_LIST} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD}"
 }
 
+
+
 display_help_text() {
 
       echo "KokkosKernels and Kokkos configure options:"
@@ -331,6 +333,10 @@ display_help_text() {
       echo "--gcc-toolchain=/Path/To/GccRoot:  Set the gcc toolchain to use with clang (e.g. /usr)" 
       echo "--kokkos-make-j=[NUM]:        Set -j parallel level for kokkos install"
       echo "                                Default: j == 4"
+      echo "--enable-tests: build Kokkos Kernels unit and performance tests"
+      echo "--enable-perfsuite: build Kokkos Kernels performance tests with
+RAJAPerf Suite"
+
 
 }
 
@@ -339,6 +345,7 @@ KOKKOS_INSTALL_PATH=""
 KOKKOS_DO_TESTS=OFF
 KOKKOS_DO_EXAMPLES=OFF
 KOKKOSKERNELS_DO_TESTS=ON
+KOKKOSKERNELS_DO_PERFSUITE=OFF
 KOKKOSKERNELS_DO_EXAMPLES=ON
 
 KOKKOS_MAKEINSTALL_J=4
@@ -491,6 +498,13 @@ do
       # This is the default
       KOKKOSKERNELS_DO_TESTS=ON
       ;;
+    --enable-perfsuite)
+      KOKKOSKERNELS_DO_PERFSUITE=ON
+      ;;
+    --disable-perfsuite)
+      # This is the default
+      KOKKOSKERNELS_DO_PERFSUITE=OFF
+      ;;
     --disable-tests)
       KOKKOSKERNELS_DO_TESTS=OFF
       ;;
@@ -744,7 +758,7 @@ cd $STORE_KOKKOSKERNELS_BUILD_PATH
 
 # Configure kokkos-kernels
 echo ""
-echo cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
+echo cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS_AND_PERFSUITE=${KOKKOSKERNELS_DO_PERFSUITE} -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
 echo ""
-cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
+cmake $COMPILER_CMD -DKokkos_DIR="${KOKKOS_FIND_PATH}" -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_INSTALL_PREFIX="${PREFIX}" -DKokkosKernels_ENABLE_TESTS_AND_PERFSUITE=${KOKKOSKERNELS_DO_PERFSUITE} -DKokkosKernels_ENABLE_TESTS=${KOKKOSKERNELS_DO_TESTS} -DKokkosKernels_ENABLE_EXAMPLES:BOOL=${KOKKOSKERNELS_DO_EXAMPLES} ${KOKKOSKERNELS_SCALARS_CMD} ${KOKKOSKERNELS_ORDINALS_CMD} ${KOKKOSKERNELS_OFFSETS_CMD} ${KOKKOSKERNELS_LAYOUTS_CMD} ${KOKKOSKERNELS_TPLS_CMD} ${KOKKOSKERNELS_USER_TPL_PATH_CMD} ${KOKKOSKERNELS_USER_TPL_LIBNAME_CMD} ${KOKKOSKERNELS_EXTRA_LINKER_FLAGS_CMD} ${KOKKOSKERNELS_BUILDTYPE_CMD} ${KOKKOSKERNELS_SPACES_CMD} ${KERNELS_DEFAULT_ETI_OPTION} ${KOKKOSKERNELS_PATH}
 

cmake/KokkosKernels_config.h.in

@@ -22,6 +22,9 @@
    than just BLAS and LAPACK functions.  */
 #cmakedefine HAVE_KOKKOSKERNELS_MKL
 
+
+#cmakedefine KOKKOSKERNELS_ENABLE_TESTS_AND_PERFSUITE
+
 /* Define this macro if experimental features of Kokkoskernels are enabled */
 #cmakedefine HAVE_KOKKOSKERNELS_EXPERIMENTAL
 

cmake/kokkoskernels_tribits.cmake

@@ -152,6 +152,11 @@ IF (IS_ENABLED)
       TESTONLYLIBS ${PARSE_TESTONLYLIBS})
   ELSE()
     ADD_EXECUTABLE(${EXE_NAME} ${PARSE_SOURCES})
+    #AJP, BMK altered:
+    IF(KOKKOSKERNELS_ENABLE_TESTS_AND_PERFSUITE)
+      TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE common ${PARSE_TESTONLYLIBS})
+    ENDIF()
+
     IF (PARSE_TESTONLYLIBS)
       TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE Kokkos::kokkoskernels ${PARSE_TESTONLYLIBS})
     ELSE ()

perf_test/CMakeLists.txt

@@ -3,29 +3,43 @@ KOKKOSKERNELS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR})
 
 KOKKOSKERNELS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../test_common)
 
-# At some point, we may wish to make this into real "performance
-# tests," in the sense that they can pass or fail.  At that point, use
-# "CATEGORIES PERFORMANCE" to mark them as such.  For now, we just
-# build the executables for manual use, but don't run the tests.  They
-# build correctly with or without MPI, but only run them with a single
-# MPI process.
+#At some point, we may wish to make this into real "performance
+#tests, " in the sense that they can pass or fail.  At that point, use
+#"CATEGORIES PERFORMANCE" to mark them as such.For now, we just
+#build the executables for manual use, but don't run the tests.  They
+#build correctly with or without MPI, but only run them with a single
+#MPI process.
 
 SET(GTEST_SOURCE_DIR ${PACKAGE_SOURCE_DIR}/tpls/gtest)
 
 KOKKOSKERNELS_ADD_TEST_LIBRARY(
-  kokkoskernelsperf_gtest
-  HEADERS ${GTEST_SOURCE_DIR}/gtest/gtest.h
-  SOURCES ${GTEST_SOURCE_DIR}/gtest/gtest-all.cc
-  )
-# Disables pthreads, this is a problem for serial builds in Trilinos & Sierra if it's enabled.
+                               kokkoskernelsperf_gtest
+                               HEADERS ${GTEST_SOURCE_DIR}/gtest/gtest.h
+                               SOURCES ${GTEST_SOURCE_DIR}/gtest/gtest-all.cc
+                                )
+#Disables pthreads, this is a problem for serial builds in Trilinos & Sierra if it's enabled.
+
 TARGET_COMPILE_DEFINITIONS(kokkoskernelsperf_gtest PUBLIC "-DGTEST_HAS_PTHREAD=0")
 TARGET_INCLUDE_DIRECTORIES(kokkoskernelsperf_gtest PUBLIC $<BUILD_INTERFACE:${GTEST_SOURCE_DIR}>)
 
-#Gtest minimally requires C++11
+#Gtest minimally requires C++ 11
 TARGET_COMPILE_FEATURES(kokkoskernelsperf_gtest PUBLIC cxx_std_11)
-
-
-
+include_directories(sparse)
+if (KokkosKernels_ENABLE_TESTS_AND_PERFSUITE)
+#Add RPS implementations of KK perf tests here
+
+    KOKKOSKERNELS_ADD_EXECUTABLE(
+        tracked_testing
+        SOURCES KokkosKernelsTrackedTesting.cpp
+        sparse/KokkosSparse_spmv_test.cpp
+        blas/blas2/KokkosBlas2_gemv_tracked_perf_test.cpp
+        blas/blas1/KokkosBlas_dot_tracked_perf_test.cpp
+        blas/blas1/KokkosBlas_team_dot_tracked_perf_test.cpp
+        PerfTestUtilities.cpp
+        sparse/spmv/OpenMPSmartStatic_SPMV.cpp
+        #sparse / KokkosSparse_spgemm_test.cpp
+        )
+endif()
 ADD_COMPONENT_SUBDIRECTORY(batched)
 ADD_COMPONENT_SUBDIRECTORY(graph)
 ADD_COMPONENT_SUBDIRECTORY(sparse)

perf_test/KokkosKernelsTrackedTesting.cpp

@@ -0,0 +1,72 @@
+//
+// Created by Poliakoff, David Zoeller on 4/26/21.
+//
+#include <common/RAJAPerfSuite.hpp>
+#include <common/Executor.hpp>
+#include "sparse/tracked_testing.hpp"
+#include <iostream>
+#include <Kokkos_Core.hpp>
+// For RPS version of BLAS Level-1 Tests
+#include "blas/blas1/tracked_testing.hpp"
+#include "blas/blas2/tracked_testing.hpp"
+int main(int argc, char* argv[]) {
+  {
+    // argument parsing for setting input data at runtime
+
+    std::string inputDataPath;
+    if (argc == 1) {
+      //    print_help();
+      std::cout << "Please provide input data directory: --input-data "
+                   "/PATH/TO/KOKKOS-KERNELS/INPUT/DATA"
+                << std::endl;
+      return 0;
+    }
+
+    for (int i = 0; i < argc; i++) {
+      // if((strcmp(argv[i],"-v")==0)) {numVecs=atoi(argv[++i]); continue;}
+      if ((strcmp(argv[i], "--input-data") == 0)) {
+        i++;
+
+        if (i == argc) {
+          std::cerr << "Must pass desired input data after '--input-data'";
+          exit(1);
+        }
+        inputDataPath = std::string(argv[i]);
+        continue;
+      }
+    }
+
+    test::set_input_data_path(inputDataPath);
+
+    // set up Executor
+    rajaperf::Executor exec(0, argv);
+    // rajaperf::Executor exec(argc, argv);
+    rajaperf::RunParams run_params(0, argv);
+    // Initialize Kokkos
+    Kokkos::initialize(argc, argv);
+
+    Kokkos::print_configuration(std::cout);
+
+    // sparse , spmv
+    test::sparse::build_executor(exec, argc, argv, run_params);
+
+    // All BLAS tests (Dot, Team Dot)
+    test::blas::build_blas_executor(exec, argc, argv, run_params);
+
+    test::blas2::build_blas2_executor(exec, argc, argv, run_params);
+
+    exec.setupSuite();
+
+    // STEP 3: Report suite run summary
+    //         (enable users to catch errors before entire suite is run)
+    exec.reportRunSummary(std::cout);
+
+    // STEP 4: Execute suite
+    exec.runSuite();
+
+    // STEP 5: Generate suite execution reports
+    exec.outputRunData();
+  }
+  Kokkos::finalize();
+  return 0;
+}

perf_test/PerfTestUtilities.cpp

@@ -0,0 +1,15 @@
+//
+// Created by Poliakoff, David Zoeller on 4/27/21.
+//
+
+#include <string>
+
+namespace test {
+
+std::string inputDataPath;
+
+void set_input_data_path(const std::string& path_to_data) {
+  inputDataPath = path_to_data;
+}
+std::string get_input_data_path() { return inputDataPath; }
+}  // namespace test

perf_test/PerfTestUtilities.hpp

@@ -0,0 +1,118 @@
+//
+// Created by Poliakoff, David Zoeller on 4/27/21.
+//
+
+#ifndef KOKKOSKERNELS_PERFTESTUTILITIES_HPP
+#define KOKKOSKERNELS_PERFTESTUTILITIES_HPP
+#include "KokkosKernels_default_types.hpp"
+#include "KokkosKernels_config.h"
+#include "KokkosKernels_IOUtils.hpp"
+#include <common/RunParams.hpp>
+#include <common/QuickKernelBase.hpp>
+#include <common/KernelBase.hpp>
+#include <dirent.h>
+
+namespace test {
+void set_input_data_path(const std::string &path_to_data);
+
+std::string get_input_data_path();
+
+}  // namespace test
+
+namespace KokkosSparse {
+
+template <class Scalar, class Ordinal, class ExecutionSpace, class,
+          class Offset>
+class CrsMatrix;
+}
+
+// helper function for get_directories
+inline bool isDirectory(std::string path) {
+  DIR *dirp;  // Pointer to a directory
+  dirp = opendir(path.c_str());
+  // bool var indicating that dirp is not NULL, i.e., a true statement
+  bool isDir = dirp != NULL;
+  if (dirp != NULL) closedir(dirp);
+  return isDir;
+}
+
+/** TODO: fix in C++17 */
+inline std::vector<std::string> get_directories(std::string path) {
+  DIR *d;
+  std::vector<std::string> paths;
+  struct dirent *dir;
+  d = opendir(path.c_str());
+  if (d) {
+    while ((dir = readdir(d)) != NULL) {
+      std::string nname = std::string(dir->d_name);
+      // Check to see if item is a directory
+      //if (isDirectory(path + '/' + nname))
+      if(nname != "." && nname != ".." && isDirectory(path + '/' + dir->d_name))
+        // std::vector::emplace_back: insert a new element to the end of vector
+        paths.emplace_back(dir->d_name);
+    }
+    closedir(d);
+  }
+  return paths;
+}
+
+namespace readers {
+
+template <class Scalar, class Ordinal, class ExecutionSpace, class Offset>
+using matrix_type =
+    KokkosSparse::CrsMatrix<Scalar, Ordinal, ExecutionSpace, void, Offset>;
+
+template <class>
+struct test_reader;
+
+template <class Scalar, class Ordinal, class ExecutionSpace, class Offset>
+struct test_reader<matrix_type<Scalar, Ordinal, ExecutionSpace, Offset>> {
+  static matrix_type<Scalar, Ordinal, ExecutionSpace, Offset> read(
+      const std::string &filename) {
+    return KokkosKernels::Impl::read_kokkos_crst_matrix<
+        matrix_type<Scalar, Ordinal, ExecutionSpace, Offset>>(filename.c_str());
+  }
+};
+
+}  // namespace readers
+template <class... SubComponents>
+struct data_retriever {
+  std::string root_path;
+  std::string sub_path;
+  struct test_case {
+    std::string filename;
+    std::tuple<SubComponents...> test_data;
+  };
+  std::vector<test_case> test_cases;
+  std::string make_full_path_to_data_file(std::string repo,
+                                          std::string path_to_data,
+                                          std::string dataset,
+                                          std::string filename) {
+    return root_path + "/" + repo + "/" + path_to_data + dataset + "/" +
+           filename;
+  }
+  template <class... Locations>
+  data_retriever(std::string path_to_data, Locations... locations)
+      : sub_path(path_to_data) {
+    root_path = test::get_input_data_path();
+
+    // TODO: way to list the directories in the root path
+    std::vector<std::string> data_repos = get_directories(root_path + "/");
+    // TODO: list directories in subpaths
+    for (auto repo : data_repos) {
+      std::vector<std::string> datasets =
+          get_directories(root_path + "/" + repo + "/" + path_to_data + "/");
+      for (auto dataset : datasets) {
+        test_cases.push_back(
+            test_case{repo + "/" + dataset,
+                      std::make_tuple(readers::test_reader<SubComponents>::read(
+                          make_full_path_to_data_file(
+                              repo, path_to_data, dataset, locations))...)});
+      }
+    }
+  }
+};
+
+using test_list = std::vector<rajaperf::KernelBase *>;
+
+#endif  // KOKKOSKERNELS_PERFTESTUTILITIES_HPP