[hannk] Add a --csv flag to compare_vs_tflite (#6149)

* [hannk] Add optional taskset support to the run_on_device scripts * [hannk] Add a --csv flag to compare_vs_tflite This lets us output results in CSV format for easy copy/paste into (eg) spreadsheets.
halide · Jul 21, 2021 · b68393c · b68393c
1 parent 025a9b9
commit b68393c
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 11 deletions.
diff --git a/apps/hannk/run_benchmark_on_device.sh b/apps/hannk/run_benchmark_on_device.sh
@@ -20,6 +20,10 @@ DEVICE_DIR="/data/local/tmp/halide/benchmarking"
 if [[ -n "${ANDROID_SERIAL}" ]]; then
   echo Using ANDROID_SERIAL=${ANDROID_SERIAL}
 fi
+if [[ -n "${TASKSET}" ]]; then
+  echo Using TASKSET=${TASKSET}
+  TASKSET_CMD="taskset ${TASKSET}"
+fi
 echo Using HL_TARGET=${HL_TARGET}
 
 if [ "$#" -eq 0 ]; then
@@ -67,7 +71,7 @@ adb shell rm -rf "${DEVICE_DIR}"
 adb shell mkdir -p "${DEVICE_DIR}"
 
 adb push ${BUILD_TARGETS} ${LOCAL_FILES} ${DEVICE_DIR}/
-adb shell "${DEVICE_DIR}/benchmark ${DEVICE_ARGS}"
+adb shell ${TASKSET_CMD} "${DEVICE_DIR}/benchmark ${DEVICE_ARGS}"
 
 echo
 echo All benchmarks complete.
diff --git a/apps/hannk/run_compare_on_device.sh b/apps/hannk/run_compare_on_device.sh
@@ -23,6 +23,10 @@ DEVICE_DIR=/data/local/tmp/halide/compare_vs_tflite
 if [[ -n "${ANDROID_SERIAL}" ]]; then
     echo Using ANDROID_SERIAL=${ANDROID_SERIAL}
 fi
+if [[ -n "${TASKSET}" ]]; then
+  echo Using TASKSET=${TASKSET}
+  TASKSET_CMD="taskset ${TASKSET}"
+fi
 echo Using HL_TARGET=${HL_TARGET}
 
 if [ "$#" -eq 0 ]; then
@@ -71,7 +75,7 @@ adb shell mkdir -p "${DEVICE_DIR}"
 
 adb push ${BUILD_TARGETS} ${TFLITE_SHARED_LIBRARY} ${LOCAL_FILES} ${DEVICE_DIR}/
 
-adb shell LD_LIBRARY_PATH=${DEVICE_DIR}:${LD_LIBRARY_PATH} ${DEVICE_DIR}/compare_vs_tflite ${DEVICE_ARGS}
+adb shell LD_LIBRARY_PATH=${DEVICE_DIR}:${LD_LIBRARY_PATH} ${TASKSET_CMD} ${DEVICE_DIR}/compare_vs_tflite ${DEVICE_ARGS}
 
 echo
 echo All comparisons complete.
diff --git a/apps/hannk/util/model_runner.cpp b/apps/hannk/util/model_runner.cpp
@@ -468,6 +468,7 @@ bool ModelRunner::compare_results(const std::string &name_a, const std::string &
         CompareBuffersOptions options;
         options.close_thresh = std::ceil((1ull << tflite_buf.type().bits) * tolerance);
         options.max_diffs_to_log = 8;
+        options.verbose = !csv_output;
         CompareBuffersResult r = dynamic_type_dispatch<CompareBuffers>(tflite_buf.type(), tflite_buf, halide_buf, options);
         if (r.ok) {
             if (verbosity >= 2) {
@@ -501,6 +502,10 @@ int ModelRunner::parse_flags(int argc, char **argv, std::vector<std::string> &fi
              this->do_compare_results = std::stoi(value) != 0;
              return 0;
          }},
+        {"csv", [this](const std::string &value) {
+             this->csv_output = std::stoi(value) != 0;
+             return 0;
+         }},
         {"enable", [this](const std::string &value) {
              for (int i = 0; i < ModelRunner::kNumRuns; i++) {
                  this->do_run[i] = false;
@@ -571,19 +576,47 @@ int ModelRunner::parse_flags(int argc, char **argv, std::vector<std::string> &fi
 }
 
 void ModelRunner::run(const std::string &filename) {
-    std::cout << "Processing " << filename << " ...\n";
+    std::map<WhichRun, RunResult> results;
 
-    const std::vector<char> buffer = read_entire_file(filename);
+    if (run_count == 0) {
+        run_count++;
 
-    std::map<WhichRun, RunResult> results;
+        for (int i = 0; i < kNumRuns; i++) {
+            if (do_run[i]) {
+                active_runs.push_back((WhichRun)i);
+            }
+        }
 
-    std::vector<WhichRun> active_runs;
-    for (int i = 0; i < kNumRuns; i++) {
-        if (do_run[i]) {
-            active_runs.push_back((WhichRun)i);
+        if (csv_output) {
+            // Output column headers
+            std::cout << "Filename";
+            if (do_benchmark) {
+                for (WhichRun i : active_runs) {
+                    std::cout << ',' << RunNames[i] << "_time_us";
+                }
+            }
+            if (do_compare_results && do_run[kTfLite]) {
+                for (WhichRun i : active_runs) {
+                    if (i == kTfLite) {
+                        continue;
+                    }
+                    std::cout << ',' << RunNames[i] << "_matches_tflite";
+                }
+            }
+            std::cout << '\n';
         }
     }
 
+    if (csv_output) {
+        // Try to print just the filename rather than a full pathname
+        const auto n = filename.rfind('/');
+        std::cout << (n == std::string::npos ? filename : filename.substr(n + 1));
+    } else {
+        std::cout << "Processing " << filename << " ...\n";
+    }
+
+    const std::vector<char> buffer = read_entire_file(filename);
+
     const auto exec_tflite = [this, &buffer]() {
         return run_in_tflite(buffer);
     };
@@ -617,7 +650,12 @@ void ModelRunner::run(const std::string &filename) {
     // ----- Log benchmark times
     if (do_benchmark) {
         for (WhichRun i : active_runs) {
-            std::cout << RunNames[i] << " Time: " << std::chrono::duration_cast<std::chrono::microseconds>(results[i].time).count() << " us\n";
+            const auto t = std::chrono::duration_cast<std::chrono::microseconds>(results[i].time).count();
+            if (csv_output) {
+                std::cout << ',' << t;
+            } else {
+                std::cout << RunNames[i] << " Time: " << t << " us\n";
+            }
         }
     }
 
@@ -628,7 +666,11 @@ void ModelRunner::run(const std::string &filename) {
             if (i == kTfLite) {
                 continue;
             }
-            if (!compare_results(RunNames[kTfLite], RunNames[i], results[kTfLite], results[i])) {
+            const bool matched = compare_results(RunNames[kTfLite], RunNames[i], results[kTfLite], results[i]);
+            if (csv_output) {
+                std::cout << ',' << (matched ? '1' : '0');
+            }
+            if (!matched) {
                 all_matched = false;
             }
         }
@@ -637,6 +679,10 @@ void ModelRunner::run(const std::string &filename) {
             exit(1);
         }
     }
+
+    if (csv_output) {
+        std::cout << '\n';
+    }
 }
 
 }  // namespace hannk
diff --git a/apps/hannk/util/model_runner.h b/apps/hannk/util/model_runner.h
@@ -103,7 +103,10 @@ struct ModelRunner {
     bool do_compare_results = true;
     bool keep_going = false;
     double tolerance;
+    bool csv_output = false;
+    int run_count = 0;
     std::string external_delegate_path;
+    std::vector<WhichRun> active_runs;
 
     ModelRunner();