Merged commit includes the following changes:

398049518 by Waymo Research: Add support for tf 2.6.0 -- 398022899 by Waymo Research: Store detection metrics details as repeated field so that different frames' results can be maintained. -- 397235629 by Waymo Research: Add custom iou functor to API -- 397103341 by Waymo Research: Add option for measurement details -- 396744255 by Waymo Research: Internal change -- 395271718 by Waymo Research: Add custom IOU calculation. -- PiperOrigin-RevId: 398049518
DavidsShi · Sep 21, 2021 · 052f50f · 052f50f
1 parent f6b42f1
commit 052f50f
Show file tree

Hide file tree

Showing 11 changed files with 304 additions and 37 deletions.
diff --git a/pip_pkg_scripts/setup.py.in b/pip_pkg_scripts/setup.py.in
@@ -18,7 +18,7 @@ from setuptools import setup
 from setuptools.command.install import install
 from setuptools.dist import Distribution
 
-__version__ = '1.4.0'
+__version__ = '1.4.1'
 REQUIRED_PACKAGES = [
     'tensorflow==' + '.'.join('TF_VERSION'.split('-')),
 ]

diff --git a/tf/workspace_tf2_6_0.bzl b/tf/workspace_tf2_6_0.bzl
@@ -0,0 +1,51 @@
+"""Install dependencies."""
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def load_tf_version_specific_dependencies():
+    """Load TF version specific dependencies."""
+    http_archive(
+        name = "com_google_absl",
+        sha256 = "35f22ef5cb286f09954b7cc4c85b5a3f6221c9d4df6b8c4a1e9d399555b366ee",
+        strip_prefix = "abseil-cpp-997aaf3a28308eba1b9156aa35ab7bca9688e9f6",
+        urls = [
+            "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/997aaf3a28308eba1b9156aa35ab7bca9688e9f6.tar.gz",
+            "https://github.com/abseil/abseil-cpp/archive/997aaf3a28308eba1b9156aa35ab7bca9688e9f6.tar.gz",
+        ],
+    )
+
+    http_archive(
+        name = "zlib",
+        build_file = "//third_party:zlib.BUILD",
+        sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+        strip_prefix = "zlib-1.2.11",
+        urls = [
+            "http://mirror.tensorflow.org/zlib.net/zlib-1.2.11.tar.gz",
+            "https://zlib.net/zlib-1.2.11.tar.gz",
+        ],
+    )
+
+    PROTOBUF_URLS = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
+        "https://github.com/protocolbuffers/protobuf/archive/v3.9.2.zip",
+    ]
+
+    PROTOBUF_SHA256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b"
+
+    PROTOBUF_STRIP_PREFIX = "protobuf-3.9.2"
+
+    # We need to import the protobuf library under the names com_google_protobuf
+    # and com_google_protobuf_cc to enable proto_library support in bazel.
+    http_archive(
+        name = "com_google_protobuf",
+        sha256 = PROTOBUF_SHA256,
+        strip_prefix = PROTOBUF_STRIP_PREFIX,
+        urls = PROTOBUF_URLS,
+    )
+
+    http_archive(
+        name = "com_google_protobuf_cc",
+        sha256 = PROTOBUF_SHA256,
+        strip_prefix = PROTOBUF_STRIP_PREFIX,
+        urls = PROTOBUF_URLS,
+    )
diff --git a/waymo_open_dataset/metrics/detection_metrics.cc b/waymo_open_dataset/metrics/detection_metrics.cc
@@ -45,23 +45,36 @@ DetectionMeasurement ComputeDetectionMeasurementFromMatchingResult(
   int num_false_positives = 0;
   int num_false_negatives = 0;
   float sum_heading_accuracy = 0.0;
+
+  DetectionMeasurement measurement;
+  DetectionMeasurement::Details* details = nullptr;
+  if (config.include_details_in_measurements()) {
+    details = measurement.add_details();
+  }
+
   auto is_in_breakdown = [&matcher, &breakdown](int gt_subset_id) {
     return internal::IsInBreakdown(
         matcher.ground_truths()[matcher.ground_truth_subset()[gt_subset_id]],
         breakdown);
   };
   for (int i = 0, sz = pd_matches.size(); i < sz; ++i) {
+    const int pd_index = matcher.prediction_subset()[i];
+    const std::string& pd_id = matcher.predictions()[pd_index].object().id();
     // This is a true positive only if
     // 1) This prediction matches a ground truth.
     // 2) The matched ground truth is in the given breakdown.
     if (internal::IsTP(pd_matches, i) &&
         (!internal::IsGroundTruthOnlyBreakdown(breakdown) ||
          is_in_breakdown(pd_matches[i]))) {
-      const float heading_accuracy = internal::ComputeHeadingAccuracy(
-          matcher, matcher.prediction_subset()[i],
-          matcher.ground_truth_subset()[pd_matches[i]]);
+      const int gt_index = matcher.ground_truth_subset()[pd_matches[i]];
+      const float heading_accuracy =
+          internal::ComputeHeadingAccuracy(matcher, pd_index, gt_index);
       if (heading_accuracy <= config.min_heading_accuracy()) continue;
       ++num_true_positives;
+      if (details != nullptr) {
+        details->add_tp_pr_ids(pd_id);
+        details->add_tp_gt_ids(matcher.ground_truths()[gt_index].object().id());
+      }
       sum_heading_accuracy += heading_accuracy;
     }
     // This is a false positive only if
@@ -70,17 +83,24 @@ DetectionMeasurement ComputeDetectionMeasurementFromMatchingResult(
     //  not inside this breakdown. The threshold of deciding whether there is an
     //  overlap is set to kOverlapIoUThreshold for now.
     if (internal::IsFP(matcher, pd_matches, i)) {
+      bool is_fp = false;
       if (internal::IsGroundTruthOnlyBreakdown(breakdown)) {
         static constexpr double kOverlapIoUThreshold = 0.01;
         const int gt_subset_id = internal::FindGTWithLargestIoU(
             matcher, i, /*iou_threshold=*/kOverlapIoUThreshold);
         const bool overlap_with_gt_in_other_shard =
             gt_subset_id >= 0 && !is_in_breakdown(gt_subset_id);
         if (!overlap_with_gt_in_other_shard) {
-          ++num_false_positives;
+          is_fp = true;
         }
       } else {
+        is_fp = true;
+      }
+      if (is_fp) {
         ++num_false_positives;
+        if (details != nullptr) {
+          details->add_fp_ids(pd_id);
+        }
       }
     }
   }
@@ -92,13 +112,22 @@ DetectionMeasurement ComputeDetectionMeasurementFromMatchingResult(
         (!internal::IsGroundTruthOnlyBreakdown(breakdown) ||
          is_in_breakdown(i))) {
       ++num_false_negatives;
+      if (details != nullptr) {
+        const int gt_index = matcher.ground_truth_subset()[i];
+        details->add_fn_ids(matcher.ground_truths()[gt_index].object().id());
+      }
     }
   }
-  DetectionMeasurement measurement;
   measurement.set_num_tps(num_true_positives);
   measurement.set_num_fps(num_false_positives);
   measurement.set_num_fns(num_false_negatives);
   measurement.set_sum_ha(sum_heading_accuracy);
+  if (details != nullptr &&
+      details->tp_gt_ids_size() != details->tp_pr_ids_size()) {
+    LOG(FATAL) << "True positive sizes should be equal. pr size: "
+               << details->tp_pr_ids_size()
+               << ", gt size: " << details->tp_gt_ids_size();
+  }
   return measurement;
 }
 
@@ -157,13 +186,22 @@ DetectionMeasurement MergeDetectionMeasurement(const DetectionMeasurement& m1,
   if (!m2.has_score_cutoff()) return m1;
   CHECK_EQ(m1.score_cutoff(), m2.score_cutoff());
   DetectionMeasurement m;
+
 #define ADD_FIELD(FIELD_NAME) \
   m.set_##FIELD_NAME(m1.FIELD_NAME() + m2.FIELD_NAME())
   ADD_FIELD(num_fps);
   ADD_FIELD(num_tps);
   ADD_FIELD(num_fns);
   ADD_FIELD(sum_ha);
 #undef ADD_FIELD
+
+  // If we enables details population, appends it as a new frame. The new
+  // frame's `details()` size should be 1.
+  if (m1.details_size() == 1 || m2.details_size() == 1) {
+    *m.mutable_details() = m1.details();
+    m.mutable_details()->MergeFrom(m2.details());
+  }
+
   m.set_score_cutoff(m1.score_cutoff());
   return m;
 }
@@ -277,11 +315,14 @@ DetectionMetrics ToDetectionMetrics(const Config& config,
 
 std::vector<DetectionMeasurements> ComputeDetectionMeasurements(
     const Config& config, const std::vector<Object>& pds,
-    const std::vector<Object>& gts) {
+    const std::vector<Object>& gts, ComputeIoUFunc custom_iou_func) {
   CHECK_GT(config.score_cutoffs_size(), 0)
       << "config.scores() must be populated: " << config.DebugString();
   std::unique_ptr<Matcher> matcher = Matcher::Create(config);
   matcher->SetGroundTruths(gts);
+  if (custom_iou_func != nullptr) {
+    matcher->SetCustomIoUComputeFunc(custom_iou_func);
+  }
   // Matcher stores a pointer to pds, so make a copy so pds lives the lifetime
   // of the matcher.
   auto pds_copy = pds;
@@ -317,7 +358,8 @@ std::vector<DetectionMeasurements> ComputeDetectionMeasurements(
 
 std::vector<DetectionMetrics> ComputeDetectionMetrics(
     const Config& config, const std::vector<std::vector<Object>>& pds,
-    const std::vector<std::vector<Object>>& gts) {
+    const std::vector<std::vector<Object>>& gts,
+    ComputeIoUFunc custom_iou_func) {
   std::vector<DetectionMeasurements> measurements;
   CHECK_EQ(pds.size(), gts.size());
   const int num_frames = pds.size();
@@ -326,10 +368,12 @@ std::vector<DetectionMetrics> ComputeDetectionMetrics(
                                  : EstimateScoreCutoffs(config, pds, gts);
   for (int i = 0; i < num_frames; ++i) {
     if (i == 0) {
-      measurements = ComputeDetectionMeasurements(config_copy, pds[i], gts[i]);
+      measurements = ComputeDetectionMeasurements(config_copy, pds[i], gts[i],
+                                                  custom_iou_func);
     } else {
       MergeDetectionMeasurementsVector(
-          ComputeDetectionMeasurements(config_copy, pds[i], gts[i]),
+          ComputeDetectionMeasurements(config_copy, pds[i], gts[i],
+                                       custom_iou_func),
           &measurements);
     }
   }

diff --git a/waymo_open_dataset/metrics/detection_metrics.h b/waymo_open_dataset/metrics/detection_metrics.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "waymo_open_dataset/metrics/matcher.h"
 #include "waymo_open_dataset/protos/metrics.pb.h"
 
 namespace waymo {
@@ -26,6 +27,7 @@ namespace open_dataset {
 // Computes detection measurements for a single frame.
 // pds: the predicted objects.
 // gts: the ground truths.
+// custom_iou_func: an optional callback for customizing IoU calculation.
 // The output vector is ordered as:
 // [{generator_i_shard_j_difficulty_level_k}].
 // i \in [0, num_breakdown_generators).
@@ -36,7 +38,7 @@ namespace open_dataset {
 //   Requires: config.score_thresholds() is populated.
 std::vector<DetectionMeasurements> ComputeDetectionMeasurements(
     const Config& config, const std::vector<Object>& pds,
-    const std::vector<Object>& gts);
+    const std::vector<Object>& gts, ComputeIoUFunc custom_iou_func = nullptr);
 
 // Computes detection metrics from measurements.
 // Each element of `measurements` is an output of ComputeDetectionMeasurements.
@@ -56,6 +58,7 @@ std::vector<DetectionMetrics> ComputeDetectionMetrics(
 // Computes detection metrics for multiple frames.
 // pds: the predicted objects.
 // gts: the ground truths.
+// custom_iou_func: an optional callback for customizing IoU calculation.
 // The output vector is ordered as:
 // [{generator_i_shard_j_difficulty_level_k}].
 // i \in [0, num_breakdown_generators).
@@ -66,7 +69,8 @@ std::vector<DetectionMetrics> ComputeDetectionMetrics(
 // Requires: 'pds', 'gts' have the same dim-0 size.
 std::vector<DetectionMetrics> ComputeDetectionMetrics(
     const Config& config, const std::vector<std::vector<Object>>& pds,
-    const std::vector<std::vector<Object>>& gts);
+    const std::vector<std::vector<Object>>& gts,
+    ComputeIoUFunc custom_iou_func = nullptr);
 
 // Estimates the score cutoffs that evenly sample the P/R curve.
 // pds: the predicted objects.