Merge branch gob.google3 into gob.master.

DavidsShi · Apr 25, 2022 · ada80c7 · ada80c7
2 parents 520c34a + 478b648
commit ada80c7
Show file tree

Hide file tree

Showing 11 changed files with 162 additions and 75 deletions.
diff --git a/README.md b/README.md
@@ -9,6 +9,7 @@ We released v1.3.1 of the Perception dataset to support the 2022 Challenges and
  - Added metrics (LET-3D-APL and LET-3D-AP) for the 3D Camera-Only Detection Challenge.
  - Added 80 segments of 20-second camera imagery, as a test set for the 3D Camera-Only Detection Challenge.
  - Added z-axis speed and acceleration in [lidar label metadata](waymo_open_dataset/label.proto#L53-L60).
+ - Fixed some inconsistencies in `projected_lidar_labels` in [dataset.proto](waymo_open_dataset/dataset.proto).
  - Updated the default configuration for the Occupancy and Flow Challenge, switching from aggregate waypoints to [subsampled waypoints](waymo_open_dataset/protos/occupancy_flow_metrics.proto#L38-L55).
  - Updated the [tutorial](waymo-open-dataset/tutorial/tutorial_3d_semseg.ipynb) for 3D Semantic Segmentation Challenge with more detailed instructions.
 

diff --git a/pip_pkg_scripts/setup.py.in b/pip_pkg_scripts/setup.py.in
@@ -18,7 +18,7 @@ from setuptools import setup
 from setuptools.command.install import install
 from setuptools.dist import Distribution
 
-__version__ = '1.4.4'
+__version__ = '1.4.5'
 REQUIRED_PACKAGES = [
     'tensorflow==' + '.'.join('TF_VERSION'.split('-')),
     'plotly>=5.5.0',

diff --git a/third_party/camera/camera_model.cc b/third_party/camera/camera_model.cc
@@ -362,7 +362,11 @@ bool CameraModel::WorldToImageMovingPointWithDepth(double x, double y, double z,
   // because the readout time per scan line is in the order of 1e-5 seconds.
   // Of course this number varies with the image size as well.
   constexpr double kThreshold = 1e-5;  // seconds.
+  // The maximum number of iterations.
   constexpr size_t kMaxIterNum = 4;
+  // Theshold for the residual capture time as a sanity check for solver
+  // divergence.
+  const double kMaxFinalResidualError = 0.1;  // seconds.
 
   Eigen::Vector2d normalized_coord;
   double residual = 2 * kThreshold;
@@ -387,6 +391,11 @@ bool CameraModel::WorldToImageMovingPointWithDepth(double x, double y, double z,
     ++iter_num;
   }
 
+  // Check sanity of final residual in case solver diverged.
+  if (std::abs(residual) > kMaxFinalResidualError) {
+    return false;
+  }
+
   // Get normalized coordinate.
   if (!ComputeDepthResidualAndJacobian(n_pos_f, n_vel_f, t_h, &normalized_coord,
                                        &point_depth, &residual,

diff --git a/tutorial/tutorial_occupancy_flow.ipynb b/tutorial/tutorial_occupancy_flow.ipynb
@@ -30,7 +30,7 @@
         "To run a Jupyter kernel locally, run:\n",
         "\n",
         "```\n",
-        "$ pip install \"waymo_open_dataset_tf_2_6_0==1.4.4\"\n",
+        "$ pip install \"waymo_open_dataset_tf_2_6_0==1.4.5\"\n",
         "$ pip install \"notebook\u003e=5.3\"\n",
         "$ jupyter notebook\n",
         "```"

diff --git a/waymo_open_dataset/metrics/breakdown_generator.cc b/waymo_open_dataset/metrics/breakdown_generator.cc
@@ -114,6 +114,9 @@ class BreakdownGeneratorRange : public BreakdownGenerator {
                           object.object().box().center_z());
     constexpr float kNearRange = 30.0;
     constexpr float kMidRange = 50.0;
+    if (object.object().type() == Label::TYPE_UNKNOWN) {
+      return -1;
+    }
     const int shard_offset = 3 * (object.object().type() - 1);
     if (shard_offset < 0) {
       return -1;
@@ -165,6 +168,9 @@ class BreakdownGeneratorSize : public BreakdownGenerator {
         std::max(object.object().box().length(), object.object().box().width()),
         object.object().box().height());
     constexpr float kLarge = 7.0;
+    if (object.object().type() == Label::TYPE_UNKNOWN) {
+      return -1;
+    }
     const int shard_offset = 2 * (object.object().type() - 1);
     if (shard_offset < 0) {
       return -1;
@@ -225,6 +231,9 @@ class BreakdownGeneratorVelocity : public BreakdownGenerator {
     constexpr float kSlow = 1.;
     constexpr float kMedium = 3.;
     constexpr float kFast = 10.;
+    if (object.object().type() == Label::TYPE_UNKNOWN) {
+      return -1;
+    }
     const int shard_offset = 5 * (object.object().type() - 1);
 
     if (v_mag < kStationary) {
@@ -241,6 +250,9 @@ class BreakdownGeneratorVelocity : public BreakdownGenerator {
   }
 
   std::vector<int> ShardsForMatching(const Object& object) const override {
+    if (object.object().type() == Label::TYPE_UNKNOWN) {
+      return std::vector<int>();
+    }
     std::vector<int> shards(5);
     for (int i = 0; i < 5; ++i) {
       shards[i] = 5 * (object.object().type() - 1) + i;
@@ -288,6 +300,9 @@ class BreakdownGeneratorCamera : public BreakdownGenerator {
   ~BreakdownGeneratorCamera() override {}
 
   int Shard(const Object& object) const override {
+    if (object.object().type() == Label::TYPE_UNKNOWN) {
+      return -1;
+    }
     const int shard_offset = kNumCameras * (object.object().type() - 1);
     CameraName::Name camera_name;
     if (Is2DCamera(object)) {
@@ -322,6 +337,9 @@ class BreakdownGeneratorCamera : public BreakdownGenerator {
   }
 
   std::vector<int> ShardsForMatching(const Object& object) const override {
+    if (object.object().type() == Label::TYPE_UNKNOWN) {
+      return std::vector<int>();
+    }
     if (Is2DCamera(object)) {
       return std::vector<int>({Shard(object)});
     }
@@ -386,20 +404,26 @@ class BreakdownGeneratorCamera : public BreakdownGenerator {
       case CameraName::SIDE_RIGHT:
         return 4;
       default:
-        return -1;
+        LOG(FATAL) << "Code should not reach here.";
     }
   }
 
   std::vector<CameraName::Name> EstimateCameraNamesByFov(
       const Object& object) const {
     std::vector<CameraName::Name> camera_names;
     constexpr double kRadToDegree = 180.0 / M_PI;
+    // The field-of-view of the camera images is approximately +-25.2 degrees.
     constexpr double kHalfFovDeg = 25.2;
+    // Considering the vehicle coordinate system, where the x-axis represents
+    // the front direction, the y-axis represents the left direction, we
+    // estimate the angle of an object to the x-axis by calculating arctan(y/x).
+    // Objects on the left side yield a positive angle, and objects on the right
+    // side yield a negative angle.
     constexpr double kFrontCameraDeg = 0.0;
-    constexpr double kFrontLeftCameraDeg = -45.0;
-    constexpr double kFrontRightCameraDeg = 45.0;
-    constexpr double kSideLeftCameraDeg = -90.0;
-    constexpr double kSideRightCameraDeg = 90.0;
+    constexpr double kFrontLeftCameraDeg = 45.0;
+    constexpr double kFrontRightCameraDeg = -45.0;
+    constexpr double kSideLeftCameraDeg = 90.0;
+    constexpr double kSideRightCameraDeg = -90.0;
 
     const double c_x = object.object().box().center_x();
     const double c_y = object.object().box().center_y();

diff --git a/waymo_open_dataset/metrics/breakdown_generator_test.cc b/waymo_open_dataset/metrics/breakdown_generator_test.cc
@@ -87,37 +87,61 @@ TEST(BreakdownGenerator, BreakdownGeneratorVelocity) {
 TEST(BreakdownGenerator, BreakdownGeneratorCamera) {
   const auto generator = BreakdownGenerator::Create(Breakdown::CAMERA);
   EXPECT_EQ(4 * 5, generator->NumShards());
-  Object object;
-  object.mutable_object()->set_type(Label::TYPE_VEHICLE);
-  object.set_camera_name(CameraName::FRONT);
-  EXPECT_EQ(0, generator->Shard(object));
-  object.Clear();
-  object.mutable_object()->set_type(Label::TYPE_PEDESTRIAN);
-  object.set_camera_name(CameraName::FRONT_LEFT);
-  EXPECT_EQ(5 + 1, generator->Shard(object));
-  object.Clear();
-  object.mutable_object()->set_type(Label::TYPE_CYCLIST);
-  object.set_camera_name(CameraName::SIDE_RIGHT);
-  EXPECT_EQ(15 + 4, generator->Shard(object));
-  object.Clear();
-  object.mutable_object()->set_type(Label::TYPE_PEDESTRIAN);
-  object.mutable_object()->set_most_visible_camera_name("FRONT_RIGHT");
-  EXPECT_EQ(5 + 2, generator->Shard(object));
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_VEHICLE);
+    object.set_camera_name(CameraName::FRONT);
+    EXPECT_EQ(0, generator->Shard(object));
+  }
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_PEDESTRIAN);
+    object.set_camera_name(CameraName::FRONT_LEFT);
+    EXPECT_EQ(5 + 1, generator->Shard(object));
+  }
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_CYCLIST);
+    object.set_camera_name(CameraName::SIDE_RIGHT);
+    EXPECT_EQ(15 + 4, generator->Shard(object));
+  }
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_PEDESTRIAN);
+    object.mutable_object()->set_most_visible_camera_name("FRONT_RIGHT");
+    EXPECT_EQ(5 + 2, generator->Shard(object));
+  }
 
-  object.Clear();
-  object.mutable_object()->set_type(Label::TYPE_VEHICLE);
-  *object.mutable_object()->mutable_box() =
-      BuildBox3d(100, 0, 0, 10, 10, 10, 0);
-  EXPECT_EQ(std::vector<int>({0}), generator->ShardsForMatching(object));
-  object.Clear();
-  object.mutable_object()->set_type(Label::TYPE_CYCLIST);
-  *object.mutable_object()->mutable_box() =
-      BuildBox3d(10, 100, 0, 10, 10, 10, 0);
-  EXPECT_EQ(std::vector<int>({15 + 4}), generator->ShardsForMatching(object));
-  object.Clear();
-  object.mutable_object()->set_type(Label::TYPE_PEDESTRIAN);
-  object.set_camera_name(CameraName::SIDE_LEFT);
-  EXPECT_EQ(std::vector<int>({5 + 3}), generator->ShardsForMatching(object));
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_VEHICLE);
+    *object.mutable_object()->mutable_box() =
+        BuildBox3d(/*center_x=*/100, /*center_y=*/0, /*center_z=*/0,
+                   /*length=*/10, /*width=*/10, /*heght=*/10, /*heading=*/0);
+    EXPECT_EQ(std::vector<int>({0}), generator->ShardsForMatching(object));
+  }
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_VEHICLE);
+    *object.mutable_object()->mutable_box() =
+        BuildBox3d(/*center_x=*/0, /*center_y=*/100, /*center_z=*/0,
+                   /*length=*/10, /*width=*/10, /*heght=*/10, /*heading=*/0);
+    EXPECT_EQ(std::vector<int>({0 + 3}), generator->ShardsForMatching(object));
+  }
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_CYCLIST);
+    *object.mutable_object()->mutable_box() =
+        BuildBox3d(/*center_x=*/100, /*center_y=*/-100, /*center_z=*/0,
+                   /*length=*/10, /*width=*/10, /*heght=*/10, /*heading=*/0);
+    EXPECT_EQ(std::vector<int>({15 + 2}), generator->ShardsForMatching(object));
+  }
+  {
+    Object object;
+    object.mutable_object()->set_type(Label::TYPE_PEDESTRIAN);
+    object.set_camera_name(CameraName::SIDE_LEFT);
+    EXPECT_EQ(std::vector<int>({5 + 3}), generator->ShardsForMatching(object));
+  }
 }
 }  // namespace
 }  // namespace open_dataset

diff --git a/waymo_open_dataset/metrics/metrics_utils.cc b/waymo_open_dataset/metrics/metrics_utils.cc
@@ -129,6 +129,8 @@ std::vector<BreakdownShardSubset> BuildSubsets(
         const std::vector<int> shards =
             breakdown_generator->ShardsForMatching(objects[i]);
         for (int s : shards) {
+          CHECK_GE(s, 0);
+          CHECK_LT(s, num_shards);
           breakdown_subsets[s].push_back(i);
         }
       } else {

diff --git a/waymo_open_dataset/metrics/tools/create_prediction_file_example.py b/waymo_open_dataset/metrics/tools/create_prediction_file_example.py
@@ -1,4 +1,3 @@
-# Lint as: python3
 # Copyright 2020 The Waymo Open Dataset Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/waymo_open_dataset/metrics/tools/create_segmentation_prediction_file_example.py b/waymo_open_dataset/metrics/tools/create_segmentation_prediction_file_example.py
@@ -1,4 +1,3 @@
-# Lint as: python3
 # Copyright 2022 The Waymo Open Dataset Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

diff --git a/waymo_open_dataset/protos/occupancy_flow_metrics.proto b/waymo_open_dataset/protos/occupancy_flow_metrics.proto
@@ -88,6 +88,14 @@ message OccupancyFlowTaskConfig {
 // Please refer to occupancy_flow_metrics.py for an implementation of these
 // metrics.
 message OccupancyFlowMetrics {
+  // The metrics stored in this proto are averages over all waypoints.  However
+  // blank waypoints are excluded when computing the metrics.  The following
+  // fields record the number of waypoints which are used for computing each
+  // of the 3 categories of metrics.
+  optional int32 num_waypoints_with_observed_occupancy = 8 [default = 0];
+  optional int32 num_waypoints_with_occluded_occupancy = 9 [default = 0];
+  optional int32 num_waypoints_with_flow = 10 [default = 0];
+
   // Treating occupancy in each grid cell as an independent binary prediction,
   // this metric measures the area under the precision-recall curve of all
   // grid cells in the future occupancy of currently-observed vehicles.

diff --git a/waymo_open_dataset/utils/occupancy_flow_metrics.py b/waymo_open_dataset/utils/occupancy_flow_metrics.py
@@ -57,6 +57,13 @@ def compute_occupancy_flow_metrics(
       pred_waypoints=pred_waypoints,
   )
 
+  metrics = occupancy_flow_metrics_pb2.OccupancyFlowMetrics()
+
+  # Flow for waypoint 0 can be constructed if there are any occupancies at
+  # waypoint 0.  -1 maps to the current time.
+  has_true_observed_occupancy = {-1: True}
+  has_true_occluded_occupancy = {-1: True}
+
   # Iterate over waypoints.
   for k in range(config.num_waypoints):
     true_observed_occupancy = true_waypoints.vehicles.observed_occupancy[k]
@@ -66,47 +73,59 @@ def compute_occupancy_flow_metrics(
     true_flow = true_waypoints.vehicles.flow[k]
     pred_flow = pred_waypoints.vehicles.flow[k]
 
+    has_true_observed_occupancy[k] = tf.reduce_max(true_observed_occupancy) > 0
+    has_true_occluded_occupancy[k] = tf.reduce_max(true_occluded_occupancy) > 0
+    has_true_flow = (has_true_observed_occupancy[k] and
+                     has_true_observed_occupancy[k - 1]) or (
+                         has_true_occluded_occupancy[k] and
+                         has_true_occluded_occupancy[k - 1])
+
     # Compute occupancy metrics.
-    metrics_dict['vehicles_observed_auc'].append(
-        _compute_occupancy_auc(true_observed_occupancy,
-                               pred_observed_occupancy))
-    metrics_dict['vehicles_occluded_auc'].append(
-        _compute_occupancy_auc(true_occluded_occupancy,
-                               pred_occluded_occupancy))
-    metrics_dict['vehicles_observed_iou'].append(
-        _compute_occupancy_soft_iou(true_observed_occupancy,
-                                    pred_observed_occupancy))
-    metrics_dict['vehicles_occluded_iou'].append(
-        _compute_occupancy_soft_iou(true_occluded_occupancy,
-                                    pred_occluded_occupancy))
+    if has_true_observed_occupancy[k]:
+      metrics.num_waypoints_with_observed_occupancy += 1
+      metrics_dict['vehicles_observed_auc'].append(
+          _compute_occupancy_auc(true_observed_occupancy,
+                                 pred_observed_occupancy))
+      metrics_dict['vehicles_observed_iou'].append(
+          _compute_occupancy_soft_iou(true_observed_occupancy,
+                                      pred_observed_occupancy))
+    if has_true_occluded_occupancy[k]:
+      metrics.num_waypoints_with_occluded_occupancy += 1
+      metrics_dict['vehicles_occluded_auc'].append(
+          _compute_occupancy_auc(true_occluded_occupancy,
+                                 pred_occluded_occupancy))
+      metrics_dict['vehicles_occluded_iou'].append(
+          _compute_occupancy_soft_iou(true_occluded_occupancy,
+                                      pred_occluded_occupancy))
 
     # Compute flow metrics.
-    metrics_dict['vehicles_flow_epe'].append(
-        _compute_flow_epe(true_flow, pred_flow))
-
-    # Compute flow-warped occupancy metrics.
-    # First, construct ground-truth occupancy of all observed and occluded
-    # vehicles.
-    true_all_occupancy = tf.clip_by_value(
-        true_observed_occupancy + true_occluded_occupancy, 0, 1)
-    # Construct predicted version of same value.
-    pred_all_occupancy = tf.clip_by_value(
-        pred_observed_occupancy + pred_occluded_occupancy, 0, 1)
-    # We expect to see the same results by warping the flow-origin occupancies.
-    flow_warped_origin_occupancy = warped_flow_origins[k]
-    # Construct quantity that requires both prediction paths to be correct.
-    flow_grounded_pred_all_occupancy = (
-        pred_all_occupancy * flow_warped_origin_occupancy)
-    # Now compute occupancy metrics between this quantity and ground-truth.
-    metrics_dict['vehicles_flow_warped_occupancy_auc'].append(
-        _compute_occupancy_auc(flow_grounded_pred_all_occupancy,
-                               true_all_occupancy))
-    metrics_dict['vehicles_flow_warped_occupancy_iou'].append(
-        _compute_occupancy_soft_iou(flow_grounded_pred_all_occupancy,
-                                    true_all_occupancy))
+    if has_true_flow:
+      metrics.num_waypoints_with_flow += 1
+      metrics_dict['vehicles_flow_epe'].append(
+          _compute_flow_epe(true_flow, pred_flow))
+
+      # Compute flow-warped occupancy metrics.
+      # First, construct ground-truth occupancy of all observed and occluded
+      # vehicles.
+      true_all_occupancy = tf.clip_by_value(
+          true_observed_occupancy + true_occluded_occupancy, 0, 1)
+      # Construct predicted version of same value.
+      pred_all_occupancy = tf.clip_by_value(
+          pred_observed_occupancy + pred_occluded_occupancy, 0, 1)
+      # Expect to see the same results by warping the flow-origin occupancies.
+      flow_warped_origin_occupancy = warped_flow_origins[k]
+      # Construct quantity that requires both prediction paths to be correct.
+      flow_grounded_pred_all_occupancy = (
+          pred_all_occupancy * flow_warped_origin_occupancy)
+      # Now compute occupancy metrics between this quantity and ground-truth.
+      metrics_dict['vehicles_flow_warped_occupancy_auc'].append(
+          _compute_occupancy_auc(flow_grounded_pred_all_occupancy,
+                                 true_all_occupancy))
+      metrics_dict['vehicles_flow_warped_occupancy_iou'].append(
+          _compute_occupancy_soft_iou(flow_grounded_pred_all_occupancy,
+                                      true_all_occupancy))
 
   # Compute means and return as proto message.
-  metrics = occupancy_flow_metrics_pb2.OccupancyFlowMetrics()
   metrics.vehicles_observed_auc = _mean(metrics_dict['vehicles_observed_auc'])
   metrics.vehicles_occluded_auc = _mean(metrics_dict['vehicles_occluded_auc'])
   metrics.vehicles_observed_iou = _mean(metrics_dict['vehicles_observed_iou'])
@@ -122,6 +141,8 @@ def compute_occupancy_flow_metrics(
 def _mean(tensor_list: Sequence[tf.Tensor]) -> float:
   """Compute mean value from a list of scalar tensors."""
   num_tensors = len(tensor_list)
+  if num_tensors == 0:
+    return 0
   sum_tensors = tf.math.add_n(tensor_list).numpy()
   return sum_tensors / num_tensors