Add mean average precision metric for object detection

CHANGELOG.md

@@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added Learned Perceptual Image Patch Similarity (LPIPS) ([#431](https://github.com/PyTorchLightning/metrics/issues/431))
 
 
+- Added `MAP` (mean average precision) metric to image package ([#467](https://github.com/PyTorchLightning/metrics/pull/467))
+
+
 - Added Tweedie Deviance Score ([#499](https://github.com/PyTorchLightning/metrics/pull/499))
 
 

docs/source/links.rst

@@ -37,6 +37,7 @@
 .. _MAPE implementation returns: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_percentage_error.html
 .. _mean squared logarithmic error: https://scikit-learn.org/stable/modules/model_evaluation.html#mean-squared-log-error
 .. _LPIPS: https://arxiv.org/abs/1801.03924
+.. _Mean-Average-Precision (mAP) and Mean-Average-Recall (mAR): https://jonathan-hui.medium.com/map-mean-average-precision-for-object-detection-45c121a31173
 .. _Tweedie Deviance Score: https://en.wikipedia.org/wiki/Tweedie_distribution#The_Tweedie_deviance
 .. _Permutation Invariant Training of Deep Models: https://ieeexplore.ieee.org/document/7952154
 .. _Computes the Top-label Calibration Error: https://arxiv.org/pdf/1909.10155.pdf

docs/source/references/modules.rst

@@ -396,6 +396,18 @@ SSIM
 .. autoclass:: torchmetrics.SSIM
     :noindex:
 
+*****************
+Detection Metrics
+*****************
+
+Object detection metrics can be used to evaluate the predicted detections with given groundtruth detections on images.
+
+MAP
+~~~
+
+.. autoclass:: torchmetrics.MAP
+    :noindex:
+
 ******************
 Regression Metrics
 ******************

requirements/detection.txt

@@ -0,0 +1 @@
+pycocotools>=2.0.2

requirements/test.txt

@@ -19,6 +19,7 @@ scikit-image>0.17.1
 # add extra requirements
 -r image.txt
 -r text.txt
+-r detection.txt
 -r audio.txt
 
 # audio

setup.py

@@ -26,8 +26,11 @@ def _load_py_module(fname, pkg="torchmetrics"):
 
 def _prepare_extras():
     extras = {
-        "image": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="image.txt"),
-        "text": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="text.txt"),
+        "image": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="image.txt"),  # skipcq: PYL-W0212
+        "text": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="text.txt"),  # skipcq: PYL-W0212
+        "detection": setup_tools._load_requirements(  # skipcq: PYL-W0212
+            path_dir=_PATH_REQUIRE, file_name="detection.txt"
+        ),
         "audio": setup_tools._load_requirements(path_dir=_PATH_REQUIRE, file_name="audio.txt"),
     }
     # create an 'all' keyword that install all possible denpendencies

tests/detection/test_map.py

@@ -0,0 +1,280 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import namedtuple
+
+import pytest
+import torch
+
+from tests.helpers.testers import MetricTester
+from torchmetrics.detection.map import MAP
+from torchmetrics.utilities.imports import _PYCOCOTOOLS_AVAILABLE
+
+Input = namedtuple("Input", ["preds", "target", "num_classes"])
+
+_inputs = Input(
+    preds=[
+        {
+            "boxes": torch.Tensor([[258.15, 41.29, 348.26, 243.78]]),
+            "scores": torch.Tensor([0.236]),
+            "labels": torch.IntTensor([4]),
+        },  # coco image id 42
+        {
+            "boxes": torch.Tensor([[61, 22.75, 504, 609.67], [12.66, 3.32, 268.6, 271.91]]),
+            "scores": torch.Tensor([0.318, 0.726]),
+            "labels": torch.IntTensor([3, 2]),
+        },  # coco image id 73
+        {
+            "boxes": torch.Tensor(
+                [
+                    [87.87, 276.25, 296.42, 103.18],
+                    [0, 3.66, 142.15, 312.4],
+                    [296.55, 93.96, 18.42, 58.83],
+                    [328.94, 97.05, 13.55, 25.93],
+                    [356.62, 95.47, 15.71, 52.08],
+                    [464.08, 105.09, 31.66, 41.9],
+                    [276.11, 103.84, 15.33, 46.88],
+                ]
+            ),
+            "scores": torch.Tensor([0.546, 0.3, 0.407, 0.611, 0.335, 0.805, 0.953]),
+            "labels": torch.IntTensor([4, 1, 0, 0, 0, 0, 0]),
+        },  # coco image id 74
+    ],
+    target=[
+        {
+            "boxes": torch.Tensor([[214.15, 41.29, 348.26, 243.78]]),
+            "labels": torch.IntTensor([4]),
+        },  # coco image id 42
+        {
+            "boxes": torch.Tensor(
+                [
+                    [13.0, 22.75, 535.98, 609.67],
+                    [1.66, 3.32, 268.6, 271.91],
+                ]
+            ),
+            "labels": torch.IntTensor([2, 2]),
+        },  # coco image id 73
+        {
+            "boxes": torch.Tensor(
+                [
+                    [61.87, 276.25, 296.42, 103.18],
+                    [2.75, 3.66, 159.4, 312.4],
+                    [295.55, 93.96, 18.42, 58.83],
+                    [326.94, 97.05, 13.55, 25.93],
+                    [356.62, 95.47, 15.71, 52.08],
+                    [462.08, 105.09, 31.66, 41.9],
+                    [277.11, 103.84, 15.33, 46.88],
+                ]
+            ),
+            "labels": torch.IntTensor([4, 1, 0, 0, 0, 0, 0]),
+        },  # coco image id 74
+    ],
+    num_classes=5,
+)
+
+
+def _compare_fn(preds, target) -> dict:
+    """Comparison function for map implementation.
+
+    Official pycocotools results calculated from a subset of https://github.com/cocodataset/cocoapi/tree/master/results
+        All classes
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.658
+        Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.876
+        Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.807
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.689
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.800
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.635
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.515
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.670
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.670
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.767
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.800
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.633
+
+        Class 0
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.725
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.780
+
+        Class 1
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.800
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.800
+
+        Class 2
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.454
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.450
+
+        Class 3
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = -1.000
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = -1.000
+
+        Class 4
+        Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.650
+        Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.650
+    """
+    return {
+        "map": torch.Tensor([0.658]),
+        "map_50": torch.Tensor([0.876]),
+        "map_75": torch.Tensor([0.807]),
+        "map_small": torch.Tensor([0.689]),
+        "map_medium": torch.Tensor([0.800]),
+        "map_large": torch.Tensor([0.635]),
+        "mar_1": torch.Tensor([0.515]),
+        "mar_10": torch.Tensor([0.670]),
+        "mar_100": torch.Tensor([0.670]),
+        "mar_small": torch.Tensor([0.767]),
+        "mar_medium": torch.Tensor([0.800]),
+        "mar_large": torch.Tensor([0.633]),
+        "map_per_class": torch.Tensor([0.725, 0.800, 0.454, -1.000, 0.650]),
+        "mar_100_per_class": torch.Tensor([0.780, 0.800, 0.450, -1.000, 0.650]),
+    }
+
+
+@pytest.mark.skipif(not _PYCOCOTOOLS_AVAILABLE, reason="test requires that pycocotools is installed")
+class TestMAP(MetricTester):
+    """Test the MAP metric for object detection predictions.
+
+    Results are compared to original values from the pycocotools implementation.
+    A subset of the first 10 fake predictions of the official repo is used:
+    https://github.com/cocodataset/cocoapi/blob/master/results/instances_val2014_fakebbox100_results.json
+    """
+
+    atol = 1e-3
+
+    @pytest.mark.parametrize("ddp", [True, False])
+    @pytest.mark.parametrize("dist_sync_on_step", [True, False])
+    def test_map(self, ddp, dist_sync_on_step):
+        """Test modular implementation for correctness."""
+
+        self.run_class_metric_test(
+            ddp=ddp,
+            preds=_inputs.preds,
+            target=_inputs.target,
+            metric_class=MAP,
+            sk_metric=_compare_fn,
+            dist_sync_on_step=dist_sync_on_step,
+            metric_args={"class_metrics": True},
+        )
+
+
+# noinspection PyTypeChecker
+@pytest.mark.skipif(not _PYCOCOTOOLS_AVAILABLE, reason="test requires that pycocotools is installed")
+def test_error_on_wrong_init():
+    """Test class raises the expected errors."""
+
+    MAP()  # no error
+
+    with pytest.raises(ValueError, match="Expected argument `class_metrics` to be a boolean"):
+        MAP(class_metrics=0)
+
+
+@pytest.mark.skipif(not _PYCOCOTOOLS_AVAILABLE, reason="test requires that pycocotools is installed")
+def test_error_on_wrong_input():
+    """Test class input validation."""
+
+    metric = MAP()
+
+    metric.update([], [])  # no error
+
+    with pytest.raises(ValueError, match="Expected argument `preds` to be of type List"):
+        metric.update(torch.Tensor(), [])  # type: ignore
+
+    with pytest.raises(ValueError, match="Expected argument `target` to be of type List"):
+        metric.update([], torch.Tensor())  # type: ignore
+
+    with pytest.raises(ValueError, match="Expected argument `preds` and `target` to have the same length"):
+        metric.update([{}], [{}, {}])
+
+    with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `boxes` key"):
+        metric.update(
+            [{"scores": torch.Tensor(), "labels": torch.IntTensor}],
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `scores` key"):
+        metric.update(
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor}],
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all dicts in `preds` to contain the `labels` key"):
+        metric.update(
+            [{"boxes": torch.Tensor(), "scores": torch.IntTensor}],
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all dicts in `target` to contain the `boxes` key"):
+        metric.update(
+            [
+                {
+                    "boxes": torch.Tensor(),
+                    "scores": torch.IntTensor,
+                    "labels": torch.IntTensor,
+                }
+            ],
+            [{"labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all dicts in `target` to contain the `labels` key"):
+        metric.update(
+            [
+                {
+                    "boxes": torch.Tensor(),
+                    "scores": torch.IntTensor,
+                    "labels": torch.IntTensor,
+                }
+            ],
+            [{"boxes": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all boxes in `preds` to be of type torch.Tensor"):
+        metric.update(
+            [{"boxes": [], "scores": torch.Tensor(), "labels": torch.IntTensor()}],
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all scores in `preds` to be of type torch.Tensor"):
+        metric.update(
+            [{"boxes": torch.Tensor(), "scores": [], "labels": torch.IntTensor()}],
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all labels in `preds` to be of type torch.Tensor"):
+        metric.update(
+            [{"boxes": torch.Tensor(), "scores": torch.Tensor(), "labels": []}],
+            [{"boxes": torch.Tensor(), "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all boxes in `target` to be of type torch.Tensor"):
+        metric.update(
+            [
+                {
+                    "boxes": torch.Tensor(),
+                    "scores": torch.Tensor(),
+                    "labels": torch.IntTensor(),
+                }
+            ],
+            [{"boxes": [], "labels": torch.IntTensor()}],
+        )
+
+    with pytest.raises(ValueError, match="Expected all labels in `target` to be of type torch.Tensor"):
+        metric.update(
+            [
+                {
+                    "boxes": torch.Tensor(),
+                    "scores": torch.Tensor(),
+                    "labels": torch.IntTensor(),
+                }
+            ],
+            [{"boxes": torch.Tensor(), "labels": []}],
+        )