Merge pull request #5620 from cz4rs/core-perf-tests-benchmark-conversion

Tracking performance testing: Port core performance tests

core/perf_test/BenchmarkMain.cpp

@@ -14,25 +14,22 @@
 //
 //@HEADER
 
-// Avoid deprecation warning for ICC
-#ifdef __INTEL_COMPILER
-#pragma warning(push)
-#pragma warning(disable : 1786)
 #include <benchmark/benchmark.h>
-#pragma warning(pop)
-#else
-#include <benchmark/benchmark.h>
-#endif
 
 #include "Benchmark_Context.hpp"
 #include <Kokkos_Core.hpp>
 
+#include "PerfTest_Category.hpp"
+
 int main(int argc, char** argv) {
   Kokkos::initialize(argc, argv);
   benchmark::Initialize(&argc, argv);
   benchmark::SetDefaultTimeUnit(benchmark::kSecond);
   KokkosBenchmark::add_benchmark_context(true);
 
+  (void)Test::command_line_num_args(argc);
+  (void)Test::command_line_arg(0, argv);
+
   benchmark::RunSpecifiedBenchmarks();
 
   benchmark::Shutdown();

core/perf_test/Benchmark_Context.cpp

@@ -0,0 +1,65 @@
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
+// See https://kokkos.org/LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//@HEADER
+
+#include "Benchmark_Context.hpp"
+
+namespace KokkosBenchmark {
+
+/**
+ * \brief Remove unwanted spaces and colon signs from input string. In case of
+ * invalid input it will return an empty string.
+ */
+std::string remove_unwanted_characters(const std::string& str) {
+  auto from = str.find_first_not_of(" :");
+  auto to   = str.find_last_not_of(" :");
+
+  if (from == std::string::npos || to == std::string::npos) {
+    return "";
+  }
+
+  // return extracted part of string without unwanted spaces and colon signs
+  return str.substr(from, to + 1);
+}
+
+/**
+ * \brief Extract all key:value pairs from kokkos configuration and add it to
+ * the benchmark context
+ */
+void add_kokkos_configuration(bool verbose) {
+  std::ostringstream msg;
+  Kokkos::print_configuration(msg, verbose);
+
+  // Iterate over lines returned from kokkos and extract key:value pairs
+  std::stringstream ss{msg.str()};
+  for (std::string line; std::getline(ss, line, '\n');) {
+    auto found = line.find_first_of(':');
+    if (found != std::string::npos) {
+      auto val = remove_unwanted_characters(line.substr(found + 1));
+      // Ignore line without value, for example a category name
+      if (!val.empty()) {
+        benchmark::AddCustomContext(
+            remove_unwanted_characters(line.substr(0, found)), val);
+      }
+    }
+  }
+}
+
+void add_benchmark_context(bool verbose) {
+  // Add Kokkos configuration to benchmark context data
+  add_kokkos_configuration(verbose);
+}
+
+}  // namespace KokkosBenchmark

core/perf_test/Benchmark_Context.hpp

@@ -19,77 +19,41 @@
 
 #include <string>
 
-// Avoid deprecation warning for ICC
-#ifdef __INTEL_COMPILER
-#pragma warning(push)
-#pragma warning(disable : 1786)
 #include <benchmark/benchmark.h>
-#pragma warning(pop)
-#else
-#include <benchmark/benchmark.h>
-#endif
 
 #include <Kokkos_Core.hpp>
 #include <Kokkos_Version_Info.hpp>
 
 namespace KokkosBenchmark {
 
-/// \brief Remove unwanted spaces and colon signs from input string. In case of
-/// invalid input it will return an empty string.
-std::string remove_unwanted_characters(std::string str) {
-  auto from = str.find_first_not_of(" :");
-  auto to   = str.find_last_not_of(" :");
-
-  if (from == std::string::npos || to == std::string::npos) {
-    return "";
-  }
-
-  // return extracted part of string without unwanted spaces and colon signs
-  return str.substr(from, to + 1);
-}
-
-/// \brief Extract all key:value pairs from kokkos configuration and add it to
-/// the benchmark context
-void add_kokkos_configuration(bool verbose) {
-  std::ostringstream msg;
-  Kokkos::print_configuration(msg, verbose);
-
-  // Iterate over lines returned from kokkos and extract key:value pairs
-  std::stringstream ss{msg.str()};
-  for (std::string line; std::getline(ss, line, '\n');) {
-    auto found = line.find_first_of(':');
-    if (found != std::string::npos) {
-      auto val = remove_unwanted_characters(line.substr(found + 1));
-      // Ignore line without value, for example a category name
-      if (!val.empty()) {
-        benchmark::AddCustomContext(
-            remove_unwanted_characters(line.substr(0, found)), val);
-      }
-    }
-  }
-}
+/**
+ * \brief Gather all context information and add it to benchmark context data
+ */
+void add_benchmark_context(bool verbose = false);
 
-/// \brief Add all data related to git to benchmark context
-void add_git_info() {
-  if (!Kokkos::Impl::GIT_BRANCH.empty()) {
-    benchmark::AddCustomContext("GIT_BRANCH", Kokkos::Impl::GIT_BRANCH);
-    benchmark::AddCustomContext("GIT_COMMIT_HASH",
-                                Kokkos::Impl::GIT_COMMIT_HASH);
-    benchmark::AddCustomContext("GIT_CLEAN_STATUS",
-                                Kokkos::Impl::GIT_CLEAN_STATUS);
-    benchmark::AddCustomContext("GIT_COMMIT_DESCRIPTION",
-                                Kokkos::Impl::GIT_COMMIT_DESCRIPTION);
-    benchmark::AddCustomContext("GIT_COMMIT_DATE",
-                                Kokkos::Impl::GIT_COMMIT_DATE);
-  }
+/**
+ * \brief Mark the label as a figure of merit.
+ */
+inline std::string benchmark_fom(const std::string& label) {
+  return "FOM: " + label;
 }
 
-/// \brief Gather all context information and add it to benchmark context data
-void add_benchmark_context(bool verbose = false) {
-  // Add Kokkos configuration to benchmark context data
-  add_kokkos_configuration(verbose);
-  // Add git information to benchmark context data
-  add_git_info();
+/**
+ * \brief Report throughput and amount of data processed for simple View
+ * operations
+ */
+template <class ViewType>
+void report_results(benchmark::State& state, ViewType view, int data_ratio,
+                    double time) {
+  // data processed in megabytes
+  const double data_processed = data_ratio * view.size() *
+                                sizeof(typename ViewType::value_type) /
+                                1'000'000;
+
+  state.SetIterationTime(time);
+  state.counters["MB"] = benchmark::Counter(data_processed);
+  state.counters[KokkosBenchmark::benchmark_fom("GB/s")] = benchmark::Counter(
+      data_processed / 1'000, benchmark::Counter::kIsIterationInvariantRate);
 }
 
 }  // namespace KokkosBenchmark