Skip to content

Commit

Permalink
Trying to improve convergence for kokkos autotuning
Browse files Browse the repository at this point in the history
  • Loading branch information
@khuck
khuck committed Apr 29, 2021
1 parent f5a776a commit e2252de
Show file tree
Hide file tree
Showing 3 changed files with 153 additions and 41 deletions.
68 changes: 50 additions & 18 deletions src/apex/apex_kokkos.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <vector>
#include <set>
#include <unordered_map>
#include <stdlib.h>
#include "apex.hpp"

/*
Expand Down Expand Up @@ -422,12 +423,25 @@ class KokkosSession {
public:
// EXHAUSTIVE, RANDOM, NELDER_MEAD, PARALLEL_RANK_ORDER
KokkosSession() :
window(3),
window(1),
strategy(apex_ah_tuning_strategy::NELDER_MEAD),
verbose(false),
use_history(false),
running(false),
history_file("") {
const char * verb = getenv("APEX_KOKKOS_VERBOSE");
if (verb != nullptr) {
if (strcmp(verb, "1") == 0) { verbose = true; }
if (strcmp(verb, "yes") == 0) { verbose = true; }
if (strcmp(verb, "YES") == 0) { verbose = true; }
if (strcmp(verb, "Yes") == 0) { verbose = true; }
if (strcmp(verb, "on") == 0) { verbose = true; }
if (strcmp(verb, "ON") == 0) { verbose = true; }
if (strcmp(verb, "On") == 0) { verbose = true; }
if (strcmp(verb, "true") == 0) { verbose = true; }
if (strcmp(verb, "TRUE") == 0) { verbose = true; }
if (strcmp(verb, "True") == 0) { verbose = true; }
}
}
int window;
apex_ah_tuning_strategy strategy;
Expand Down Expand Up @@ -485,20 +499,20 @@ void Variable::makeSpace(void) {
dstep = info.candidates.range.step.double_value;
dmin = info.candidates.range.lower.double_value;
dmax = info.candidates.range.upper.double_value;
if (info.candidates.range.openLower) {
if (!info.candidates.range.openLower) {
dmin = dmin + dstep;
}
if (info.candidates.range.openUpper) {
if (!info.candidates.range.openUpper) {
dmax = dmax - dstep;
}
} else if (info.type == kokkos_value_int64) {
lstep = info.candidates.range.step.int_value;
lmin = info.candidates.range.lower.int_value;
lmax = info.candidates.range.upper.int_value;
if (info.candidates.range.openLower) {
if (!info.candidates.range.openLower) {
lmin = lmin + lstep;
}
if (info.candidates.range.openUpper) {
if (!info.candidates.range.openUpper) {
lmax = lmax - lstep;
}
}
Expand Down Expand Up @@ -540,8 +554,10 @@ size_t& getDepth() {
*/
void kokkosp_declare_output_type(const char* name, const size_t id,
Kokkos_Tools_VariableInfo& info) {
std::cout << std::string(getDepth(), ' ');
std::cout << __func__ << std::endl;
if(getSession().verbose) {
std::cout << std::string(getDepth(), ' ');
std::cout << __func__ << std::endl;
}
Variable * output = new Variable(id, name, info);
output->makeSpace();
getSession().outputs.insert(std::make_pair(id, output));
Expand All @@ -557,8 +573,10 @@ void kokkosp_declare_output_type(const char* name, const size_t id,
*/
void kokkosp_declare_input_type(const char* name, const size_t id,
Kokkos_Tools_VariableInfo& info) {
std::cout << std::string(getDepth(), ' ');
std::cout << __func__ << std::endl;
if(getSession().verbose) {
std::cout << std::string(getDepth(), ' ');
std::cout << __func__ << std::endl;
}
Variable * input = new Variable(id, name, info);
getSession().inputs.insert(std::make_pair(id, input));
getSession().outputs.insert(std::make_pair(id, input));
Expand Down Expand Up @@ -595,6 +613,7 @@ std::string hashContext(size_t numVars, const Kokkos_Tools_VariableValue* values
for (size_t i = 0 ; i < numVars ; i++) {
auto id = values[i].type_id;
ss << d << id << ":";
/*
Variable* var{getSession().inputs[id]};
switch (var->info.type) {
case kokkos_value_double:
Expand All @@ -609,6 +628,8 @@ std::string hashContext(size_t numVars, const Kokkos_Tools_VariableValue* values
default:
break;
}
*/
ss << values[i].value.string_value;
d = ",";
}
ss << "]";
Expand Down Expand Up @@ -693,25 +714,31 @@ void handle_start(const std::string & name, const size_t vars,
return 0.0;
}
double result = profile->accumulated/profile->calls;
//if(session.verbose) {
fprintf(stderr, "time per call: %f\n", (double)(result)/1000000000.0);
//}
if(session.verbose) {
fprintf(stdout, "time per call: %fs\n", (double)(result)/1000000000.0);
}
return result;
};
request->set_metric(metric);

// Set apex_openmp_policy_tuning_strategy
request->set_strategy(session.strategy);
request->set_radius(0.05);
request->set_aggregation_times(3);
// min, max, mean
request->set_aggregation_function("min");

for (size_t i = 0 ; i < vars ; i++) {
auto id = values[i].type_id;
Variable* var{getSession().outputs[id]};
if (var->info.type == kokkos_value_double) {
std::cout << session.outputs[id]->name << " init: " <<
session.outputs[id]->dmin << " min: " <<
session.outputs[id]->dmin << " max: " <<
session.outputs[id]->dmax << " step: " <<
session.outputs[id]->dstep << std::endl;
if (session.verbose) {
std::cout << "\n" << session.outputs[id]->name <<
" init: " << session.outputs[id]->dmin <<
" min: " << session.outputs[id]->dmin <<
" max: " << session.outputs[id]->dmax <<
" step: " << session.outputs[id]->dstep;
}
auto tmp = request->add_param_double(
session.outputs[id]->name,
session.outputs[id]->dmin,
Expand All @@ -732,6 +759,7 @@ void handle_start(const std::string & name, const size_t vars,
session.outputs[id]->espace);
}
}
std::cout << std::endl;

// Set OpenMP runtime parameters to initial values.
set_params(request, vars, values);
Expand Down Expand Up @@ -800,8 +828,8 @@ void kokkosp_request_values(
getSession().active_requests.insert(std::pair<uint32_t, std::string>(contextId, name));
if (getSession().verbose) {
std::cout << std::endl << std::string(getDepth(), ' ');
}
printTuning(numTuningVariables, tuningVariableValues);
}
// throw away the time spent in this step!
getSession().context_starts[contextId] = apex::profiler::now_ns();
}
Expand All @@ -811,10 +839,12 @@ void kokkosp_request_values(
* starting measurement.
*/
void kokkosp_begin_context(size_t contextId) {
/*
if (getSession().verbose) {
std::cout << std::string(getDepth()++, ' ');
std::cout << __func__ << "\t" << contextId << std::endl;
}
*/
std::stringstream ss;
getSession().context_starts.insert(
std::pair<uint32_t, uint64_t>(contextId, apex::profiler::now_ns()));
Expand All @@ -825,10 +855,12 @@ void kokkosp_begin_context(size_t contextId) {
* values can now be associated with a result.
*/
void kokkosp_end_context(const size_t contextId) {
/*
if (getSession().verbose) {
std::cout << std::string(--getDepth(), ' ');
std::cout << __func__ << "\t" << contextId << std::endl;
}
*/
uint64_t end = apex::profiler::now_ns();
auto start = getSession().context_starts.find(contextId);
auto name = getSession().active_requests.find(contextId);
Expand Down
90 changes: 70 additions & 20 deletions src/apex/apex_policies.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <vector>
#include "apex_cxx_shared_lock.hpp"
#include "apex_assert.h"
#include <unistd.h>

#ifdef APEX_HAVE_RCR
#include "libenergy.h"
Expand Down Expand Up @@ -1125,10 +1126,13 @@ inline int __active_harmony_custom_setup(shared_ptr<apex_tuning_session>
return APEX_NOERROR;
}

inline int __active_harmony_custom_setup(
shared_ptr<apex_tuning_session> tuning_session,
apex_tuning_request & request) {
const char* session_name = request.name.c_str();
inline int apex_start_session (shared_ptr<apex_tuning_session> tuning_session,
const char* session_name, bool restart = false) {

/* Leave the session, if started */
if(restart && tuning_session->hdesc != nullptr) {
ah_free(tuning_session->hdesc);
}
tuning_session->hdesc = ah_alloc();
if (tuning_session->hdesc == nullptr) {
cerr << "Failed to allocate Active Harmony session" << endl;
Expand All @@ -1138,6 +1142,19 @@ inline int __active_harmony_custom_setup(
cerr << "Could not set Active Harmony session name" << endl;
return APEX_ERROR;
}
if (ah_connect(tuning_session->hdesc, nullptr, 0) != 0) {
cerr << "Failed to launch Active Harmony tuning session: " <<
endl << ah_error() << endl;
return APEX_ERROR;
}
return APEX_NOERROR;
}

inline int __active_harmony_custom_setup(
shared_ptr<apex_tuning_session> tuning_session,
apex_tuning_request & request) {
const char* session_name = request.name.c_str();
apex_start_session(tuning_session, session_name);
tuning_session->hdef = ah_def_alloc();
if(tuning_session->hdef == nullptr) {
cerr << "Could not allocate Active Harmony definition descriptor." << endl;
Expand All @@ -1156,10 +1173,21 @@ inline int __active_harmony_custom_setup(
<< library_name << endl;
return APEX_ERROR;
}
if (ah_def_cfg(tuning_session->hdef, "INIT_RADIUS",
std::to_string(request.radius).c_str()) != 0) {
cerr << "Failed to set Active Harmony tuning raduis to "
<< request.radius << endl;
std::stringstream ss;
/* Log the search */
ss << "agg.so:log.so";
std::string layers{ss.str()};
if (ah_def_layers(tuning_session->hdef, layers.c_str()) != 0) {
cerr << "Failed to set Active Harmony layers to "
<< layers << endl;
return APEX_ERROR;
}
if (ah_def_cfg(tuning_session->hdef, "LOG_FILE", "/tmp/ah_search.log") != 0) {
cerr << "Failed to set Active Harmony log file name" << endl;
return APEX_ERROR;
}
if (ah_def_cfg(tuning_session->hdef, "LOG_MODE", "w") != 0) {
cerr << "Failed to set Active Harmony log file mode" << endl;
return APEX_ERROR;
}
/* Instruct the aggregator to collect X performance values for
Expand All @@ -1173,13 +1201,16 @@ inline int __active_harmony_custom_setup(
<< request.aggregation_times << endl;
return APEX_ERROR;
}
if (ah_def_cfg(tuning_session->hdef, "AGG_FUNC", "median") != 0) {
if (ah_def_cfg(tuning_session->hdef, "AGG_FUNC",
request.aggregation_function.c_str()) != 0) {
cerr << "Failed to set Active Harmony tuning aggregation function to "
<< request.aggregation_function << endl;
return APEX_ERROR;
}
}

std::stringstream inits;
std::string delimiter{""};
for(auto & kv : request.params) {
auto & param = kv.second;
const char * param_name = param->get_name().c_str();
Expand All @@ -1195,6 +1226,8 @@ inline int __active_harmony_custom_setup(
endl;
return APEX_ERROR;
}
inits << delimiter << param_long->get_init();
delimiter = ", ";
}
break;

Expand All @@ -1209,6 +1242,8 @@ inline int __active_harmony_custom_setup(
<< endl;
return APEX_ERROR;
}
inits << delimiter << param_double->get_init();
delimiter = ", ";
}
break;

Expand All @@ -1235,6 +1270,8 @@ inline int __active_harmony_custom_setup(
return APEX_ERROR;
}
}
inits << delimiter << param_enum->get_init();
delimiter = ", ";
};
break;

Expand All @@ -1245,20 +1282,33 @@ inline int __active_harmony_custom_setup(
return APEX_ERROR;
}
}

if (ah_connect(tuning_session->hdesc, nullptr, 0) != 0) {
cerr << "Failed to launch Active Harmony tuning session: " <<
endl << ah_error() << endl;
std::string init_point{inits.str()};
if (ah_def_cfg(tuning_session->hdef, "INIT_POINT", init_point.c_str()) != 0) {
cerr << "Failed to set Active Harmony initial point to " << init_point << endl;
return APEX_ERROR;
}

tuning_session->htask = ah_start(tuning_session->hdesc, tuning_session->hdef);
if (!tuning_session->htask) {
cerr << "Failed to join Active Harmony tuning session" << endl;
cerr << ah_error() << endl;
APEX_ASSERT(false);
return APEX_ERROR;
}
do {
useconds_t sleepytime = 1000;
tuning_session->htask = ah_start(tuning_session->hdesc, tuning_session->hdef);
if (!tuning_session->htask) {
cerr << "Failed to join Active Harmony tuning session. Reason:" << endl;
cerr << ah_error() << endl;
cerr << "Retrying..." << endl;
} else {
cout << "Joined Active Harmony tuning session" << endl;
break;
}
sleepytime = sleepytime * sleepytime;
if (sleepytime > 1000000) {
APEX_ASSERT(false);
return APEX_ERROR;
}
usleep(sleepytime);
apex_start_session(tuning_session, session_name, true);
} while (true);

ah_def_free(tuning_session->hdef);

return APEX_NOERROR;
}
Expand Down
Loading

0 comments on commit e2252de

Please sign in to comment.