Adding random search, and adding output from exhaustive and random se…

…arch strategies for Kokkos autotuning.

src/apex/CMakeLists.hpx

@@ -309,6 +309,7 @@ set(apex_headers
     profiler.hpp
     profile_reducer.hpp
     profiler_listener.hpp
+    random.hpp
     semaphore.hpp
     simulated_annealing.hpp
     thread_instance.hpp
@@ -340,6 +341,7 @@ set(apex_sources
     policy_handler.cpp
     profile_reducer.cpp
     profiler_listener.cpp
+    random.cpp
     simulated_annealing.cpp
     task_identifier.cpp
     tau_listener.cpp

src/apex/CMakeLists.standalone

@@ -108,6 +108,7 @@ policy_handler.cpp
 ${PROC_SOURCE}
 profile_reducer.cpp
 profiler_listener.cpp
+random.cpp
 ${SENSOR_SOURCE}
 simulated_annealing.cpp
 task_identifier.cpp
@@ -188,6 +189,7 @@ INSTALL(FILES apex.h
     handler.hpp
     memory_wrapper.hpp
     profile.hpp
+    random.hpp
     apex_export.h
     utils.hpp
     apex_options.hpp

src/apex/apex_kokkos_tuning.cpp

@@ -211,15 +211,25 @@ class KokkosSession {
 private:
 // EXHAUSTIVE, RANDOM, NELDER_MEAD, PARALLEL_RANK_ORDER
     KokkosSession() :
-        window(5),
-        //strategy(apex_ah_tuning_strategy::APEX_EXHAUSTIVE),
-        strategy(apex_ah_tuning_strategy::SIMULATED_ANNEALING),
-        //strategy(apex_ah_tuning_strategy::NELDER_MEAD),
+        window(apex::apex_options::kokkos_tuning_window()),
         verbose(false),
         use_history(false),
         running(false){
             verbose = apex::apex_options::use_kokkos_verbose();
             // don't do this until the object is constructed!
+            saveCache = (apex::apex::instance()->get_node_id() == 0) ? true : false;
+            if (strncmp(apex::apex_options::kokkos_tuning_policy(),
+                    "random", strlen("random")) == 0) {
+                strategy = apex_ah_tuning_strategy::APEX_RANDOM;
+            } else if (strncmp(apex::apex_options::kokkos_tuning_policy(),
+                    "exhaustive", strlen("exhaustive")) == 0) {
+                strategy = apex_ah_tuning_strategy::APEX_EXHAUSTIVE;
+            } else if (strncmp(apex::apex_options::kokkos_tuning_policy(),
+                    "simulated_annealing", strlen("simulated_annealing")) == 0) {
+                strategy = apex_ah_tuning_strategy::SIMULATED_ANNEALING;
+            } else {
+                strategy = apex_ah_tuning_strategy::NELDER_MEAD;
+            }
     }
 public:
     ~KokkosSession() {
@@ -229,13 +239,14 @@ class KokkosSession {
     KokkosSession(const KokkosSession&) =delete;
     KokkosSession& operator=(const KokkosSession&) =delete;
     int window;
+    bool verbose;
+    bool use_history;
+    bool running;
+    bool saveCache;
     apex_ah_tuning_strategy strategy;
     std::unordered_map<std::string, std::shared_ptr<apex_tuning_request>>
         requests;
     std::unordered_map<std::string, std::vector<int>> var_ids;
-    bool verbose;
-    bool use_history;
-    bool running;
     std::map<size_t, Variable*> inputs;
     std::map<size_t, Variable*> outputs;
     apex_policy_handle * start_policy_handle;
@@ -314,7 +325,7 @@ void KokkosSession::saveOutputVar(size_t id, Variable * var) {
 
 void KokkosSession::writeCache(void) {
     if(use_history) { return; }
-    if(apex::apex::instance()->get_node_id() != 0) { return; }
+    if(!saveCache) { return; }
     // did the user specify a file?
     if (strlen(apex::apex_options::kokkos_tuning_cache()) > 0) {
         cacheFilename = std::string(apex::apex_options::kokkos_tuning_cache());
@@ -340,9 +351,12 @@ void KokkosSession::writeCache(void) {
         results << "Context_" << count++ << ":" << std::endl;
         results << "  Name: \"" << req.first << "\"" << std::endl;
         std::shared_ptr<apex_tuning_request> request = req.second;
+        // always write the random search out
+        bool converged = request->has_converged() ||
+            strategy == apex_ah_tuning_strategy::APEX_RANDOM;
         results << "  Converged: " <<
-            (request->has_converged() ? "true" : "false") << std::endl;
-        if (request->has_converged()) {
+            (converged ? "true" : "false") << std::endl;
+        if (converged) {
             results << "  Results:" << std::endl;
             results << "    NumVars: " << var_ids[req.first].size() << std::endl;
             for (const auto &id : var_ids[req.first]) {

src/apex/apex_policies.cpp

@@ -936,6 +936,35 @@ int apex_exhaustive_policy(shared_ptr<apex_tuning_session> tuning_session,
     return APEX_NOERROR;
 }
 
+int apex_random_policy(shared_ptr<apex_tuning_session> tuning_session,
+    apex_context const context) {
+    APEX_UNUSED(context);
+    if (apex_final) return APEX_NOERROR; // we terminated
+    std::unique_lock<std::mutex> l{shutdown_mutex};
+    if (tuning_session->random_session.converged()) {
+        if (!tuning_session->converged_message) {
+            tuning_session->converged_message = true;
+            cout << "APEX: Tuning has converged for session " << tuning_session->id
+            << "." << endl;
+            tuning_session->random_session.saveBestSettings();
+            tuning_session->random_session.printBestSettings();
+        }
+        tuning_session->random_session.saveBestSettings();
+        return APEX_NOERROR;
+    }
+
+    // get a measurement of our current setting
+    double new_value = tuning_session->metric_of_interest();
+
+    /* Report the performance we've just measured. */
+    tuning_session->random_session.evaluate(new_value);
+
+    /* Request new settings for next time */
+    tuning_session->random_session.getNewSettings();
+
+    return APEX_NOERROR;
+}
+
 
 /// ----------------------------------------------------------------------------
 ///
@@ -1544,6 +1573,67 @@ inline int __exhaustive_setup(shared_ptr<apex_tuning_session>
   return APEX_NOERROR;
 }
 
+inline int __random_setup(shared_ptr<apex_tuning_session>
+    tuning_session, apex_tuning_request & request) {
+  APEX_UNUSED(tuning_session);
+  // set up the Simulated annealing!
+  // iterate over the parameters, and create variables.
+  using namespace apex::random;
+  for(auto & kv : request.params) {
+      auto & param = kv.second;
+      const char * param_name = param->get_name().c_str();
+      switch(param->get_type()) {
+          case apex_param_type::LONG: {
+              auto param_long =
+              std::static_pointer_cast<apex_param_long>(param);
+              Variable v(VariableType::longtype, param_long->value.get());
+              long lvalue = param_long->min;
+              do {
+                  v.lvalues.push_back(lvalue);
+                  lvalue = lvalue + param_long->step;
+              } while (lvalue < param_long->max);
+              v.set_init();
+              tuning_session->random_session.add_var(param_name, std::move(v));
+          }
+          break;
+          case apex_param_type::DOUBLE: {
+              auto param_double =
+              std::static_pointer_cast<apex_param_double>(param);
+              Variable v(VariableType::doubletype, param_double->value.get());
+              double dvalue = param_double->min;
+              do {
+                  v.dvalues.push_back(dvalue);
+                  dvalue = dvalue + param_double->step;
+              } while (dvalue < param_double->max);
+              v.set_init();
+              tuning_session->random_session.add_var(param_name, std::move(v));
+          }
+          break;
+          case apex_param_type::ENUM: {
+              auto param_enum =
+              std::static_pointer_cast<apex_param_enum>(param);
+              Variable v(VariableType::stringtype, param_enum->value.get());
+              for(const std::string & possible_value :
+                             param_enum->possible_values) {
+                  v.svalues.push_back(possible_value);
+              }
+              v.set_init();
+              tuning_session->random_session.add_var(param_name, std::move(v));
+          }
+          break;
+          default:
+              cerr <<
+              "ERROR: Attempted to register tuning parameter with unknown type."
+              << endl;
+              return APEX_ERROR;
+      }
+  }
+  /* request initial settings */
+  tuning_session->random_session.getNewSettings();
+
+  return APEX_NOERROR;
+}
+
 inline int __common_setup_timer_throttling(apex_optimization_criteria_t
     criteria, apex_optimization_method_t method, unsigned long update_interval)
 {
@@ -1647,6 +1737,16 @@ inline int __common_setup_custom_tuning(shared_ptr<apex_tuning_session>
             }
             );
         }
+    } else if (request.strategy == apex_ah_tuning_strategy::APEX_RANDOM) {
+        status = __random_setup(tuning_session, request);
+        if(status == APEX_NOERROR) {
+            apex::register_policy(
+            request.trigger,
+            [=](apex_context const & context)->int {
+                return apex_random_policy(tuning_session, context);
+            }
+            );
+        }
     } else {
         int status = __active_harmony_custom_setup(tuning_session, request);
         if(status == APEX_NOERROR) {

src/apex/apex_policies.hpp

@@ -31,10 +31,12 @@
 #include "simulated_annealing.hpp"
 // include the exhaustive class
 #include "exhaustive.hpp"
+// include the random class
+#include "random.hpp"
 
 enum class apex_param_type : int {NONE, LONG, DOUBLE, ENUM};
 enum class apex_ah_tuning_strategy : int {EXHAUSTIVE, RANDOM, NELDER_MEAD,
-PARALLEL_RANK_ORDER, SIMULATED_ANNEALING, APEX_EXHAUSTIVE};
+PARALLEL_RANK_ORDER, SIMULATED_ANNEALING, APEX_EXHAUSTIVE, APEX_RANDOM};
 
 struct apex_tuning_session;
 class apex_tuning_request;
@@ -71,6 +73,8 @@ class apex_param {
             tuning_session, apex_tuning_request & request);
         friend int __exhaustive_setup(std::shared_ptr<apex_tuning_session>
             tuning_session, apex_tuning_request & request);
+        friend int __random_setup(std::shared_ptr<apex_tuning_session>
+            tuning_session, apex_tuning_request & request);
 };
 
 class apex_param_long : public apex_param {
@@ -106,6 +110,8 @@ class apex_param_long : public apex_param {
             tuning_session, apex_tuning_request & request);
         friend int __exhaustive_setup(std::shared_ptr<apex_tuning_session>
             tuning_session, apex_tuning_request & request);
+        friend int __random_setup(std::shared_ptr<apex_tuning_session>
+            tuning_session, apex_tuning_request & request);
 };
 
 class apex_param_double : public apex_param {
@@ -141,6 +147,8 @@ class apex_param_double : public apex_param {
             tuning_session, apex_tuning_request & request);
         friend int __exhaustive_setup(std::shared_ptr<apex_tuning_session>
             tuning_session, apex_tuning_request & request);
+        friend int __random_setup(std::shared_ptr<apex_tuning_session>
+            tuning_session, apex_tuning_request & request);
 };
 
 class apex_param_enum : public apex_param {
@@ -175,6 +183,8 @@ class apex_param_enum : public apex_param {
             tuning_session, apex_tuning_request & request);
         friend int __exhaustive_setup(std::shared_ptr<apex_tuning_session>
             tuning_session, apex_tuning_request & request);
+        friend int __random_setup(std::shared_ptr<apex_tuning_session>
+            tuning_session, apex_tuning_request & request);
 };
 
 
@@ -313,6 +323,8 @@ class apex_tuning_request {
             tuning_session, apex_tuning_request & request);
         friend int __exhaustive_setup(std::shared_ptr<apex_tuning_session>
             tuning_session, apex_tuning_request & request);
+        friend int __random_setup(std::shared_ptr<apex_tuning_session>
+            tuning_session, apex_tuning_request & request);
 };
 
 
@@ -337,6 +349,8 @@ struct apex_tuning_session {
     apex::simulated_annealing::SimulatedAnnealing sa_session;
     // if using exhaustive, this is the request.
     apex::exhaustive::Exhaustive exhaustive_session;
+    // if using exhaustive, this is the request.
+    apex::random::Random random_session;
     bool converged_message = false;
 
     // variables related to power throttling

src/apex/apex_types.h

@@ -340,6 +340,7 @@ inline unsigned int sc_nprocessors_onln()
     macro (APEX_KOKKOS_VERBOSE, use_kokkos_verbose, bool, false, "Enable verbose Kokkos autotuning and profiling messages.") \
     macro (APEX_KOKKOS_COUNTERS, use_kokkos_counters, bool, false, "Enable Kokkos counters.") \
     macro (APEX_KOKKOS_TUNING, use_kokkos_tuning, bool, true, "Enable Kokkos autotuning.") \
+    macro (APEX_KOKKOS_TUNING_WINDOW, kokkos_tuning_window, int, 5, "Minimum number of tests per candidate while autotuning.") \
     macro (APEX_KOKKOS_PROFILING_FENCES, use_kokkos_profiling_fences, bool, false, "Force Kokkos to fence after all Kokkos kernel launches (recommended, but not required).") \
     macro (APEX_START_DELAY_SECONDS, start_delay_seconds, int, 0, "Delay collection of APEX data for N seconds.") \
     macro (APEX_MAX_DURATION_SECONDS, max_duration_seconds, int, 0, "Collect APEX data for only N seconds.") \
@@ -361,6 +362,7 @@ inline unsigned int sc_nprocessors_onln()
         APEX_DEFAULT_OTF2_ARCHIVE_NAME, "OTF2 trace filename.") \
     macro (APEX_EVENT_FILTER_FILE, task_event_filter_file, char*, "", "File containing names of timers to include/exclude during data collection.") \
     macro (APEX_KOKKOS_TUNING_CACHE, kokkos_tuning_cache, char*, "", "Filename contining Kokkos autotuned results, tuned offline.") \
+    macro (APEX_KOKKOS_TUNING_POLICY, kokkos_tuning_policy, char*, "simulated_annealing", "Kokkos autotuning policy: random, exhaustive, simulated_annealing, nelder_mead.") \
     macro (APEX_ROCPROF_METRICS, rocprof_metrics, char*, "", "List of metrics to periodically sample with the Rocprofiler library (see /opt/rocm/rocprofiler/lib/metrics.xml).")
     // macro (APEX_ROCPROF_METRICS, rocprof_metrics, char*, "MemUnitBusy,MemUnitStalled,VALUUtilization,VALUBusy,SALUBusy,L2CacheHit,WriteUnitStalled,ALUStalledByLDS,LDSBankConflict", "")
 

src/apex/exhaustive.cpp

@@ -1,5 +1,7 @@
 #include "exhaustive.hpp"
 #include <algorithm>
+#include <iostream>
+#include <fstream>
 
 namespace apex {
 
@@ -34,7 +36,31 @@ size_t Exhaustive::get_max_iterations() {
     return max_iter;
 }
 
+class log_wrapper {
+    private:
+        std::ofstream myfile;
+    public:
+        log_wrapper(const std::map<std::string, Variable>& vars) {
+            myfile.open("tuning.csv");
+            myfile << "iter,";
+            for (auto& v : vars) { myfile << v.first << ","; }
+            myfile << "time" << std::endl;
+        }
+        ~log_wrapper() {
+            myfile.close();
+        }
+        std::ofstream& getstream() {
+            return myfile;
+        }
+};
+
 void Exhaustive::evaluate(double new_cost) {
+    static log_wrapper log(vars);
+    static size_t count{0};
+    if (++count % 10000 == 0) { std::cout << count << std::endl; }
+    log.getstream() << count << ",";
+    for (auto& v : vars) { log.getstream() << v.second.toString() << ","; }
+    log.getstream() << new_cost << std::endl;
     if (new_cost < cost) {
         if (new_cost < best_cost) {
             best_cost = new_cost;

src/apex/exhaustive.hpp

@@ -69,6 +69,17 @@ class Variable {
         *((const char**)(value)) = svalues[best_index].c_str();
         return svalues[best_index];
     }
+    std::string toString() {
+        if (vtype == VariableType::doubletype) {
+            return std::to_string(dvalues[current_index]);
+        }
+        else if (vtype == VariableType::longtype) {
+            return std::to_string(lvalues[current_index]);
+        }
+        //else if (vtype == VariableType::stringtype) {
+        return svalues[current_index];
+        //}
+    }
 };
 
 class Exhaustive {