diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5e989af7bab..0f3768c7313 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,7 +25,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added support for float targets in `nDCG` metric ([#437](https://github.com/PyTorchLightning/metrics/pull/437))
 - Added `average` argument to `AveragePrecision` metric for reducing multi-label and multi-class problems ([#477](https://github.com/PyTorchLightning/metrics/pull/477))
 - Added `MultioutputWrapper` ([#510](https://github.com/PyTorchLightning/metrics/pull/510))
-- Added metric sweeping `higher_is_better` as constant attribute ([#544](https://github.com/PyTorchLightning/metrics/pull/544))
+- Added metric sweeping:
+  - `higher_is_better` as constant attribute ([#544](https://github.com/PyTorchLightning/metrics/pull/544))
+  - `higher_is_better` to rest of codebase ([#584](https://github.com/PyTorchLightning/metrics/pull/584))
 - Added simple aggregation metrics: `SumMetric`, `MeanMetric`, `CatMetric`, `MinMetric`, `MaxMetric` ([#506](https://github.com/PyTorchLightning/metrics/pull/506))
 - Added pairwise submodule with metrics ([#553](https://github.com/PyTorchLightning/metrics/pull/553))
   - `pairwise_cosine_similarity`
diff --git a/torchmetrics/audio/snr.py b/torchmetrics/audio/snr.py
index 3afdb93b884..46b1f217bfb 100644
--- a/torchmetrics/audio/snr.py
+++ b/torchmetrics/audio/snr.py
@@ -71,6 +71,7 @@ class SNR(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = True
     sum_snr: Tensor
     total: Tensor
 
diff --git a/torchmetrics/classification/accuracy.py b/torchmetrics/classification/accuracy.py
index 9eb30721f13..b6dec6f8036 100644
--- a/torchmetrics/classification/accuracy.py
+++ b/torchmetrics/classification/accuracy.py
@@ -165,6 +165,7 @@ class Accuracy(StatScores):
 
     """
     is_differentiable = False
+    higher_is_better = True
     correct: Tensor
     total: Tensor
 
diff --git a/torchmetrics/classification/auroc.py b/torchmetrics/classification/auroc.py
index d081b3295bc..cfc2920a923 100644
--- a/torchmetrics/classification/auroc.py
+++ b/torchmetrics/classification/auroc.py
@@ -105,6 +105,7 @@ class AUROC(Metric):
 
     """
     is_differentiable = False
+    higher_is_better = True
     preds: List[Tensor]
     target: List[Tensor]
 
diff --git a/torchmetrics/classification/calibration_error.py b/torchmetrics/classification/calibration_error.py
index c9c8f9062c5..6ece374a690 100644
--- a/torchmetrics/classification/calibration_error.py
+++ b/torchmetrics/classification/calibration_error.py
@@ -60,6 +60,7 @@ class CalibrationError(Metric):
             default: None (which selects the entire world)
     """
     DISTANCES = {"l1", "l2", "max"}
+    higher_is_better = False
     confidences: List[Tensor]
     accuracies: List[Tensor]
 
diff --git a/torchmetrics/classification/cohen_kappa.py b/torchmetrics/classification/cohen_kappa.py
index 213497a9734..6ceb391bf3a 100644
--- a/torchmetrics/classification/cohen_kappa.py
+++ b/torchmetrics/classification/cohen_kappa.py
@@ -78,6 +78,7 @@ class labels.
 
     """
     is_differentiable = False
+    higher_is_better = True
     confmat: Tensor
 
     def __init__(
diff --git a/torchmetrics/classification/f_beta.py b/torchmetrics/classification/f_beta.py
index 687eb3226e3..ca9e78c48b6 100644
--- a/torchmetrics/classification/f_beta.py
+++ b/torchmetrics/classification/f_beta.py
@@ -270,6 +270,7 @@ class F1(FBeta):
     """
 
     is_differentiable = False
+    higher_is_better = True
 
     def __init__(
         self,
diff --git a/torchmetrics/classification/hamming_distance.py b/torchmetrics/classification/hamming_distance.py
index b60e98c5f6a..4da5c601220 100644
--- a/torchmetrics/classification/hamming_distance.py
+++ b/torchmetrics/classification/hamming_distance.py
@@ -68,6 +68,7 @@ class HammingDistance(Metric):
 
     """
     is_differentiable = False
+    higher_is_better = False
     correct: Tensor
     total: Tensor
 
diff --git a/torchmetrics/classification/hinge.py b/torchmetrics/classification/hinge.py
index 5ea86287ffb..a1e7dca84a3 100644
--- a/torchmetrics/classification/hinge.py
+++ b/torchmetrics/classification/hinge.py
@@ -85,6 +85,7 @@ class Hinge(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = False
     measure: Tensor
     total: Tensor
 
diff --git a/torchmetrics/classification/iou.py b/torchmetrics/classification/iou.py
index 20db9a42600..789d57e8523 100644
--- a/torchmetrics/classification/iou.py
+++ b/torchmetrics/classification/iou.py
@@ -78,6 +78,7 @@ class IoU(ConfusionMatrix):
 
     """
     is_differentiable = False
+    higher_is_better = True
 
     def __init__(
         self,
diff --git a/torchmetrics/classification/kl_divergence.py b/torchmetrics/classification/kl_divergence.py
index 40aa9f8b6b7..cf85ee79533 100644
--- a/torchmetrics/classification/kl_divergence.py
+++ b/torchmetrics/classification/kl_divergence.py
@@ -62,6 +62,7 @@ class KLDivergence(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = False
     # TODO: canot be used because if scripting
     # measures: Union[List[Tensor], Tensor]
     total: Tensor
diff --git a/torchmetrics/classification/matthews_corrcoef.py b/torchmetrics/classification/matthews_corrcoef.py
index 096ce46a2f0..c01ac0f1f3c 100644
--- a/torchmetrics/classification/matthews_corrcoef.py
+++ b/torchmetrics/classification/matthews_corrcoef.py
@@ -74,6 +74,7 @@ class MatthewsCorrcoef(Metric):
 
     """
     is_differentiable = False
+    higher_is_better = True
     confmat: Tensor
 
     def __init__(
diff --git a/torchmetrics/classification/precision_recall.py b/torchmetrics/classification/precision_recall.py
index 206ce77da9e..627fde14a4b 100644
--- a/torchmetrics/classification/precision_recall.py
+++ b/torchmetrics/classification/precision_recall.py
@@ -121,6 +121,7 @@ class Precision(StatScores):
 
     """
     is_differentiable = False
+    higher_is_better = True
 
     def __init__(
         self,
@@ -271,6 +272,7 @@ class Recall(StatScores):
 
     """
     is_differentiable = False
+    higher_is_better = True
 
     def __init__(
         self,
diff --git a/torchmetrics/classification/specificity.py b/torchmetrics/classification/specificity.py
index ee0e6c88332..c21ccb8211a 100644
--- a/torchmetrics/classification/specificity.py
+++ b/torchmetrics/classification/specificity.py
@@ -122,6 +122,7 @@ class Specificity(StatScores):
 
     """
     is_differentiable = False
+    higher_is_better = True
 
     def __init__(
         self,
diff --git a/torchmetrics/image/fid.py b/torchmetrics/image/fid.py
index a6dadba6fe1..3b891ec3145 100644
--- a/torchmetrics/image/fid.py
+++ b/torchmetrics/image/fid.py
@@ -204,6 +204,7 @@ class FID(Metric):
     """
     real_features: List[Tensor]
     fake_features: List[Tensor]
+    higher_is_better = False
 
     def __init__(
         self,
diff --git a/torchmetrics/image/inception.py b/torchmetrics/image/inception.py
index 3d70d90939c..d2c4504c1d7 100644
--- a/torchmetrics/image/inception.py
+++ b/torchmetrics/image/inception.py
@@ -101,6 +101,7 @@ class IS(Metric):
 
     """
     features: List
+    higher_is_better = True
 
     def __init__(
         self,
diff --git a/torchmetrics/image/kid.py b/torchmetrics/image/kid.py
index ccdc362e5b2..d860a2706b8 100644
--- a/torchmetrics/image/kid.py
+++ b/torchmetrics/image/kid.py
@@ -164,6 +164,7 @@ class KID(Metric):
     """
     real_features: List[Tensor]
     fake_features: List[Tensor]
+    higher_is_better = False
 
     def __init__(
         self,
diff --git a/torchmetrics/image/lpip_similarity.py b/torchmetrics/image/lpip_similarity.py
index 7f26e358d26..2b23b8570d0 100644
--- a/torchmetrics/image/lpip_similarity.py
+++ b/torchmetrics/image/lpip_similarity.py
@@ -88,6 +88,7 @@ class LPIPS(Metric):
     """
 
     is_differentiable = True
+    higher_is_better = False
     real_features: List[Tensor]
     fake_features: List[Tensor]
 
diff --git a/torchmetrics/image/psnr.py b/torchmetrics/image/psnr.py
index 7dff5214d72..948675980a3 100644
--- a/torchmetrics/image/psnr.py
+++ b/torchmetrics/image/psnr.py
@@ -69,6 +69,7 @@ class PSNR(Metric):
     """
     min_target: Tensor
     max_target: Tensor
+    higher_is_better = False
 
     def __init__(
         self,
diff --git a/torchmetrics/image/ssim.py b/torchmetrics/image/ssim.py
index 17f05e2e9b5..e2d26855d65 100644
--- a/torchmetrics/image/ssim.py
+++ b/torchmetrics/image/ssim.py
@@ -52,6 +52,7 @@ class SSIM(Metric):
 
     preds: List[Tensor]
     target: List[Tensor]
+    higher_is_better = True
 
     def __init__(
         self,
diff --git a/torchmetrics/regression/cosine_similarity.py b/torchmetrics/regression/cosine_similarity.py
index 7721b55013b..42c1e4d9095 100644
--- a/torchmetrics/regression/cosine_similarity.py
+++ b/torchmetrics/regression/cosine_similarity.py
@@ -62,6 +62,7 @@ class CosineSimilarity(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = True
     preds: List[Tensor]
     target: List[Tensor]
 
diff --git a/torchmetrics/regression/explained_variance.py b/torchmetrics/regression/explained_variance.py
index e11b637dcfa..939a6884734 100644
--- a/torchmetrics/regression/explained_variance.py
+++ b/torchmetrics/regression/explained_variance.py
@@ -78,6 +78,7 @@ class ExplainedVariance(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = True
     n_obs: Tensor
     sum_error: Tensor
     sum_squared_error: Tensor
diff --git a/torchmetrics/regression/mean_absolute_error.py b/torchmetrics/regression/mean_absolute_error.py
index 7468db589d9..0f84270c9ad 100644
--- a/torchmetrics/regression/mean_absolute_error.py
+++ b/torchmetrics/regression/mean_absolute_error.py
@@ -49,6 +49,7 @@ class MeanAbsoluteError(Metric):
         tensor(0.5000)
     """
     is_differentiable = True
+    higher_is_better = False
     sum_abs_error: Tensor
     total: Tensor
 
diff --git a/torchmetrics/regression/mean_absolute_percentage_error.py b/torchmetrics/regression/mean_absolute_percentage_error.py
index cab503ab493..98512c4d578 100644
--- a/torchmetrics/regression/mean_absolute_percentage_error.py
+++ b/torchmetrics/regression/mean_absolute_percentage_error.py
@@ -58,6 +58,7 @@ class MeanAbsolutePercentageError(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = False
     sum_abs_per_error: Tensor
     total: Tensor
 
diff --git a/torchmetrics/regression/mean_squared_error.py b/torchmetrics/regression/mean_squared_error.py
index 791f736f3d9..34b7d0274b0 100644
--- a/torchmetrics/regression/mean_squared_error.py
+++ b/torchmetrics/regression/mean_squared_error.py
@@ -52,6 +52,7 @@ class MeanSquaredError(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = False
     sum_squared_error: Tensor
     total: Tensor
 
diff --git a/torchmetrics/regression/mean_squared_log_error.py b/torchmetrics/regression/mean_squared_log_error.py
index c42f542e677..eefbfc71b75 100644
--- a/torchmetrics/regression/mean_squared_log_error.py
+++ b/torchmetrics/regression/mean_squared_log_error.py
@@ -53,6 +53,7 @@ class MeanSquaredLogError(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = False
     sum_squared_log_error: Tensor
     total: Tensor
 
diff --git a/torchmetrics/regression/pearson.py b/torchmetrics/regression/pearson.py
index 830ee6a3c1e..60f66be2562 100644
--- a/torchmetrics/regression/pearson.py
+++ b/torchmetrics/regression/pearson.py
@@ -86,6 +86,7 @@ class PearsonCorrcoef(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = None  # both -1 and 1 are optimal
     preds: List[Tensor]
     target: List[Tensor]
     mean_x: Tensor
diff --git a/torchmetrics/regression/r2.py b/torchmetrics/regression/r2.py
index 0453d35e28d..5d67889b71e 100644
--- a/torchmetrics/regression/r2.py
+++ b/torchmetrics/regression/r2.py
@@ -88,6 +88,7 @@ class R2Score(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = True
     sum_squared_error: Tensor
     sum_error: Tensor
     residual: Tensor
diff --git a/torchmetrics/regression/spearman.py b/torchmetrics/regression/spearman.py
index 75325ef0e4a..0d0c5a6ba6b 100644
--- a/torchmetrics/regression/spearman.py
+++ b/torchmetrics/regression/spearman.py
@@ -54,6 +54,7 @@ class SpearmanCorrcoef(Metric):
 
     """
     is_differentiable = False
+    higher_is_better = True
     preds: List[Tensor]
     target: List[Tensor]
 
diff --git a/torchmetrics/regression/symmetric_mean_absolute_percentage_error.py b/torchmetrics/regression/symmetric_mean_absolute_percentage_error.py
index dc26df15a05..05cc46dad39 100644
--- a/torchmetrics/regression/symmetric_mean_absolute_percentage_error.py
+++ b/torchmetrics/regression/symmetric_mean_absolute_percentage_error.py
@@ -55,6 +55,7 @@ class SymmetricMeanAbsolutePercentageError(Metric):
         tensor(0.2290)
     """
     is_differentiable = True
+    higher_is_better = False
     sum_abs_per_error: Tensor
     total: Tensor
 
diff --git a/torchmetrics/regression/tweedie_deviance.py b/torchmetrics/regression/tweedie_deviance.py
index ed1cf4429d9..bf1d37c5f90 100644
--- a/torchmetrics/regression/tweedie_deviance.py
+++ b/torchmetrics/regression/tweedie_deviance.py
@@ -75,6 +75,7 @@ class TweedieDevianceScore(Metric):
 
     """
     is_differentiable = True
+    higher_is_better = None  # TODO: both -1 and 1 are optimal
     sum_deviance_score: Tensor
     num_observations: Tensor