Multilabel Ranking metrics

CHANGELOG.md

@@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+
+- Added `CoverageError` to classification metrics ([#787](https://github.com/PyTorchLightning/metrics/pull/787))
+
+
+- Added `LabelRankingAveragePrecision` and `LabelRankingLoss` ([#787](https://github.com/PyTorchLightning/metrics/pull/787))
+
+
 - Added support for `MetricCollection` in `MetricTracker` ([#718](https://github.com/PyTorchLightning/metrics/pull/718))
 
 

docs/source/references/functional.rst

@@ -112,6 +112,13 @@ confusion_matrix [func]
     :noindex:
 
 
+coverage_error [func]
+~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: torchmetrics.functional.coverage_error
+    :noindex:
+
+
 dice_score [func]
 ~~~~~~~~~~~~~~~~~
 
@@ -161,6 +168,20 @@ kl_divergence [func]
     :noindex:
 
 
+label_ranking_average_precision [func]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: torchmetrics.functional.label_ranking_average_precision
+    :noindex:
+
+
+label_ranking_loss [func]
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autofunction:: torchmetrics.functional.label_ranking_loss
+    :noindex:
+
+
 matthews_corrcoef [func]
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/references/modules.rst

@@ -274,6 +274,12 @@ ConfusionMatrix
 .. autoclass:: torchmetrics.ConfusionMatrix
     :noindex:
 
+CoverageError
+~~~~~~~~~~~~~
+
+.. autoclass:: torchmetrics.CoverageError
+    :noindex:
+
 F1Score
 ~~~~~~~
 
@@ -310,6 +316,18 @@ KLDivergence
 .. autoclass:: torchmetrics.KLDivergence
     :noindex:
 
+LabelRankingAveragePrecision
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: torchmetrics.LabelRankingAveragePrecision
+    :noindex:
+
+LabelRankingLoss
+~~~~~~~~~~~~~~~~
+
+.. autoclass:: torchmetrics.LabelRankingLoss
+    :noindex:
+
 MatthewsCorrCoef
 ~~~~~~~~~~~~~~~~
 

tests/classification/test_ranking.py

@@ -0,0 +1,101 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import pytest
+import torch
+from sklearn.metrics import coverage_error as sk_coverage_error
+from sklearn.metrics import label_ranking_average_precision_score as sk_label_ranking
+from sklearn.metrics import label_ranking_loss as sk_label_ranking_loss
+
+from tests.classification.inputs import _input_multilabel_logits as _input_mlb_logits
+from tests.classification.inputs import _input_multilabel_prob as _input_mlb_prob
+from tests.helpers import seed_all
+from tests.helpers.testers import BATCH_SIZE, NUM_BATCHES, MetricTester
+from torchmetrics.classification.ranking import CoverageError, LabelRankingAveragePrecision, LabelRankingLoss
+from torchmetrics.functional.classification.ranking import (
+    coverage_error,
+    label_ranking_average_precision,
+    label_ranking_loss,
+)
+
+seed_all(42)
+
+
+def _sk_coverage_error(preds, target, sample_weight=None):
+    if sample_weight is not None:
+        sample_weight = sample_weight.numpy()
+    return sk_coverage_error(target.numpy(), preds.numpy(), sample_weight=sample_weight)
+
+
+def _sk_label_ranking(preds, target, sample_weight=None):
+    if sample_weight is not None:
+        sample_weight = sample_weight.numpy()
+    return sk_label_ranking(target.numpy(), preds.numpy(), sample_weight=sample_weight)
+
+
+def _sk_label_ranking_loss(preds, target, sample_weight=None):
+    if sample_weight is not None:
+        sample_weight = sample_weight.numpy()
+    return sk_label_ranking_loss(target.numpy(), preds.numpy(), sample_weight=sample_weight)
+
+
+@pytest.mark.parametrize(
+    "metric, functional_metric, sk_metric",
+    [
+        (CoverageError, coverage_error, _sk_coverage_error),
+        (LabelRankingAveragePrecision, label_ranking_average_precision, _sk_label_ranking),
+        (LabelRankingLoss, label_ranking_loss, _sk_label_ranking_loss),
+    ],
+)
+@pytest.mark.parametrize(
+    "preds, target",
+    [
+        (_input_mlb_logits.preds, _input_mlb_logits.target),
+        (_input_mlb_prob.preds, _input_mlb_prob.target),
+    ],
+)
+@pytest.mark.parametrize("sample_weight", [None, torch.rand(NUM_BATCHES, BATCH_SIZE)])
+class TestRanking(MetricTester):
+    @pytest.mark.parametrize("ddp", [False, True])
+    @pytest.mark.parametrize("dist_sync_on_step", [False, True])
+    def test_ranking_class(
+        self, ddp, dist_sync_on_step, preds, target, metric, functional_metric, sk_metric, sample_weight
+    ):
+        self.run_class_metric_test(
+            ddp=ddp,
+            preds=preds,
+            target=target,
+            metric_class=metric,
+            sk_metric=sk_metric,
+            dist_sync_on_step=dist_sync_on_step,
+            fragment_kwargs=True,
+            sample_weight=sample_weight,
+        )
+
+    def test_ranking_functional(self, preds, target, metric, functional_metric, sk_metric, sample_weight):
+        self.run_functional_metric_test(
+            preds,
+            target,
+            metric_functional=functional_metric,
+            sk_metric=sk_metric,
+            fragment_kwargs=True,
+            sample_weight=sample_weight,
+        )
+
+    def test_ranking_differentiability(self, preds, target, metric, functional_metric, sk_metric, sample_weight):
+        self.run_differentiability_test(
+            preds=preds,
+            target=target,
+            metric_module=metric,
+            metric_functional=functional_metric,
+        )

torchmetrics/__init__.py

@@ -32,12 +32,15 @@
     CalibrationError,
     CohenKappa,
     ConfusionMatrix,
+    CoverageError,
     F1Score,
     FBetaScore,
     HammingDistance,
     HingeLoss,
     JaccardIndex,
     KLDivergence,
+    LabelRankingAveragePrecision,
+    LabelRankingLoss,
     MatthewsCorrCoef,
     Precision,
     PrecisionRecallCurve,
@@ -115,6 +118,7 @@
     "CohenKappa",
     "ConfusionMatrix",
     "CosineSimilarity",
+    "CoverageError",
     "TweedieDevianceScore",
     "ExplainedVariance",
     "ExtendedEditDistance",
@@ -124,6 +128,8 @@
     "HingeLoss",
     "JaccardIndex",
     "KLDivergence",
+    "LabelRankingAveragePrecision",
+    "LabelRankingLoss",
     "MatthewsCorrCoef",
     "MaxMetric",
     "MeanAbsoluteError",

torchmetrics/classification/__init__.py

@@ -29,6 +29,11 @@
 from torchmetrics.classification.matthews_corrcoef import MatthewsCorrCoef  # noqa: F401
 from torchmetrics.classification.precision_recall import Precision, Recall  # noqa: F401
 from torchmetrics.classification.precision_recall_curve import PrecisionRecallCurve  # noqa: F401
+from torchmetrics.classification.ranking import (  # noqa: F401
+    CoverageError,
+    LabelRankingAveragePrecision,
+    LabelRankingLoss,
+)
 from torchmetrics.classification.roc import ROC  # noqa: F401
 from torchmetrics.classification.specificity import Specificity  # noqa: F401
 from torchmetrics.classification.stat_scores import StatScores  # noqa: F401