From b68393cc467cc8c664f16cd5cfe6925ba2140555 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 21 Jul 2021 16:08:12 -0700 Subject: [PATCH] [hannk] Add a --csv flag to compare_vs_tflite (#6149) * [hannk] Add optional taskset support to the run_on_device scripts * [hannk] Add a --csv flag to compare_vs_tflite This lets us output results in CSV format for easy copy/paste into (eg) spreadsheets. --- apps/hannk/run_benchmark_on_device.sh | 6 ++- apps/hannk/run_compare_on_device.sh | 6 ++- apps/hannk/util/model_runner.cpp | 64 +++++++++++++++++++++++---- apps/hannk/util/model_runner.h | 3 ++ 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/apps/hannk/run_benchmark_on_device.sh b/apps/hannk/run_benchmark_on_device.sh index 45908a6e3381..1551fdfbb913 100755 --- a/apps/hannk/run_benchmark_on_device.sh +++ b/apps/hannk/run_benchmark_on_device.sh @@ -20,6 +20,10 @@ DEVICE_DIR="/data/local/tmp/halide/benchmarking" if [[ -n "${ANDROID_SERIAL}" ]]; then echo Using ANDROID_SERIAL=${ANDROID_SERIAL} fi +if [[ -n "${TASKSET}" ]]; then + echo Using TASKSET=${TASKSET} + TASKSET_CMD="taskset ${TASKSET}" +fi echo Using HL_TARGET=${HL_TARGET} if [ "$#" -eq 0 ]; then @@ -67,7 +71,7 @@ adb shell rm -rf "${DEVICE_DIR}" adb shell mkdir -p "${DEVICE_DIR}" adb push ${BUILD_TARGETS} ${LOCAL_FILES} ${DEVICE_DIR}/ -adb shell "${DEVICE_DIR}/benchmark ${DEVICE_ARGS}" +adb shell ${TASKSET_CMD} "${DEVICE_DIR}/benchmark ${DEVICE_ARGS}" echo echo All benchmarks complete. diff --git a/apps/hannk/run_compare_on_device.sh b/apps/hannk/run_compare_on_device.sh index 1ede7f83c0d5..cce34fbc88e0 100755 --- a/apps/hannk/run_compare_on_device.sh +++ b/apps/hannk/run_compare_on_device.sh @@ -23,6 +23,10 @@ DEVICE_DIR=/data/local/tmp/halide/compare_vs_tflite if [[ -n "${ANDROID_SERIAL}" ]]; then echo Using ANDROID_SERIAL=${ANDROID_SERIAL} fi +if [[ -n "${TASKSET}" ]]; then + echo Using TASKSET=${TASKSET} + TASKSET_CMD="taskset ${TASKSET}" +fi echo Using HL_TARGET=${HL_TARGET} if [ "$#" -eq 0 ]; then @@ -71,7 +75,7 @@ adb shell mkdir -p "${DEVICE_DIR}" adb push ${BUILD_TARGETS} ${TFLITE_SHARED_LIBRARY} ${LOCAL_FILES} ${DEVICE_DIR}/ -adb shell LD_LIBRARY_PATH=${DEVICE_DIR}:${LD_LIBRARY_PATH} ${DEVICE_DIR}/compare_vs_tflite ${DEVICE_ARGS} +adb shell LD_LIBRARY_PATH=${DEVICE_DIR}:${LD_LIBRARY_PATH} ${TASKSET_CMD} ${DEVICE_DIR}/compare_vs_tflite ${DEVICE_ARGS} echo echo All comparisons complete. diff --git a/apps/hannk/util/model_runner.cpp b/apps/hannk/util/model_runner.cpp index 534727c370b0..7fe7513c9404 100644 --- a/apps/hannk/util/model_runner.cpp +++ b/apps/hannk/util/model_runner.cpp @@ -468,6 +468,7 @@ bool ModelRunner::compare_results(const std::string &name_a, const std::string & CompareBuffersOptions options; options.close_thresh = std::ceil((1ull << tflite_buf.type().bits) * tolerance); options.max_diffs_to_log = 8; + options.verbose = !csv_output; CompareBuffersResult r = dynamic_type_dispatch(tflite_buf.type(), tflite_buf, halide_buf, options); if (r.ok) { if (verbosity >= 2) { @@ -501,6 +502,10 @@ int ModelRunner::parse_flags(int argc, char **argv, std::vector &fi this->do_compare_results = std::stoi(value) != 0; return 0; }}, + {"csv", [this](const std::string &value) { + this->csv_output = std::stoi(value) != 0; + return 0; + }}, {"enable", [this](const std::string &value) { for (int i = 0; i < ModelRunner::kNumRuns; i++) { this->do_run[i] = false; @@ -571,19 +576,47 @@ int ModelRunner::parse_flags(int argc, char **argv, std::vector &fi } void ModelRunner::run(const std::string &filename) { - std::cout << "Processing " << filename << " ...\n"; + std::map results; - const std::vector buffer = read_entire_file(filename); + if (run_count == 0) { + run_count++; - std::map results; + for (int i = 0; i < kNumRuns; i++) { + if (do_run[i]) { + active_runs.push_back((WhichRun)i); + } + } - std::vector active_runs; - for (int i = 0; i < kNumRuns; i++) { - if (do_run[i]) { - active_runs.push_back((WhichRun)i); + if (csv_output) { + // Output column headers + std::cout << "Filename"; + if (do_benchmark) { + for (WhichRun i : active_runs) { + std::cout << ',' << RunNames[i] << "_time_us"; + } + } + if (do_compare_results && do_run[kTfLite]) { + for (WhichRun i : active_runs) { + if (i == kTfLite) { + continue; + } + std::cout << ',' << RunNames[i] << "_matches_tflite"; + } + } + std::cout << '\n'; } } + if (csv_output) { + // Try to print just the filename rather than a full pathname + const auto n = filename.rfind('/'); + std::cout << (n == std::string::npos ? filename : filename.substr(n + 1)); + } else { + std::cout << "Processing " << filename << " ...\n"; + } + + const std::vector buffer = read_entire_file(filename); + const auto exec_tflite = [this, &buffer]() { return run_in_tflite(buffer); }; @@ -617,7 +650,12 @@ void ModelRunner::run(const std::string &filename) { // ----- Log benchmark times if (do_benchmark) { for (WhichRun i : active_runs) { - std::cout << RunNames[i] << " Time: " << std::chrono::duration_cast(results[i].time).count() << " us\n"; + const auto t = std::chrono::duration_cast(results[i].time).count(); + if (csv_output) { + std::cout << ',' << t; + } else { + std::cout << RunNames[i] << " Time: " << t << " us\n"; + } } } @@ -628,7 +666,11 @@ void ModelRunner::run(const std::string &filename) { if (i == kTfLite) { continue; } - if (!compare_results(RunNames[kTfLite], RunNames[i], results[kTfLite], results[i])) { + const bool matched = compare_results(RunNames[kTfLite], RunNames[i], results[kTfLite], results[i]); + if (csv_output) { + std::cout << ',' << (matched ? '1' : '0'); + } + if (!matched) { all_matched = false; } } @@ -637,6 +679,10 @@ void ModelRunner::run(const std::string &filename) { exit(1); } } + + if (csv_output) { + std::cout << '\n'; + } } } // namespace hannk diff --git a/apps/hannk/util/model_runner.h b/apps/hannk/util/model_runner.h index 130b48c135e7..e3ef8e77367b 100644 --- a/apps/hannk/util/model_runner.h +++ b/apps/hannk/util/model_runner.h @@ -103,7 +103,10 @@ struct ModelRunner { bool do_compare_results = true; bool keep_going = false; double tolerance; + bool csv_output = false; + int run_count = 0; std::string external_delegate_path; + std::vector active_runs; ModelRunner();