From 873334dadfbac98f1fc574deb68e15724b690396 Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Fri, 26 Mar 2021 11:48:58 +0100 Subject: [PATCH 1/5] ssim --- docs/source/pages/overview.rst | 1 + tests/regression/test_ssim.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/docs/source/pages/overview.rst b/docs/source/pages/overview.rst index 3f38056fe20..7bf2bf860d4 100644 --- a/docs/source/pages/overview.rst +++ b/docs/source/pages/overview.rst @@ -145,6 +145,7 @@ the following limitations: but they are also listed below: - :ref:`references/modules:PSNR` and :ref:`references/functional:psnr [func]` + - :ref:`references/modules:SSIM` and :ref:`references/functional:ssim [func]` ****************** Metric Arithmetics diff --git a/tests/regression/test_ssim.py b/tests/regression/test_ssim.py index 5d7e1d7c533..ba4c39235fe 100644 --- a/tests/regression/test_ssim.py +++ b/tests/regression/test_ssim.py @@ -90,6 +90,15 @@ def test_ssim_functional(self, preds, target, multichannel): partial(_sk_metric, data_range=1.0, multichannel=multichannel), metric_args={"data_range": 1.0}, ) + + # SSIM half + cpu does not work due to missing support in torch.log + @pytest.mark.xfail(reason="SSIM metric does not support cpu + half precision") + def test_ssim_half_cpu(self, preds, target, multichannel): + self.run_precision_test_cpu(preds, target, SSIM, ssim, {"data_range": 1.0}) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason='test requires cuda') + def test_ssim_half_gpu(self, preds, target, multichannel): + self.run_precision_test_gpu( preds, target, SSIM, ssim, {"data_range": 1.0}) @pytest.mark.parametrize( From bf49d1568780819c2f01faf84b18d13b43585a1c Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Fri, 26 Mar 2021 11:53:00 +0100 Subject: [PATCH 2/5] explained_variance --- tests/regression/test_explained_variance.py | 7 +++++++ tests/regression/test_ssim.py | 2 +- torchmetrics/functional/regression/explained_variance.py | 9 +++++---- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/regression/test_explained_variance.py b/tests/regression/test_explained_variance.py index 4548b2b010e..79d1aafa993 100644 --- a/tests/regression/test_explained_variance.py +++ b/tests/regression/test_explained_variance.py @@ -83,6 +83,13 @@ def test_explained_variance_functional(self, multioutput, preds, target, sk_metr partial(sk_metric, sk_fn=partial(explained_variance_score, multioutput=multioutput)), metric_args=dict(multioutput=multioutput), ) + + def test_explained_variance_half_cpu(self, multioutput, preds, target, sk_metric): + self.run_precision_test_cpu(preds, target, ExplainedVariance, explained_variance) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason='test requires cuda') + def test_explained_variance_half_gpu(self, multioutput, preds, target, sk_metric): + self.run_precision_test_gpu(preds, target, ExplainedVariance, explained_variance) def test_error_on_different_shape(metric_class=ExplainedVariance): diff --git a/tests/regression/test_ssim.py b/tests/regression/test_ssim.py index ba4c39235fe..b6ec163db46 100644 --- a/tests/regression/test_ssim.py +++ b/tests/regression/test_ssim.py @@ -98,7 +98,7 @@ def test_ssim_half_cpu(self, preds, target, multichannel): @pytest.mark.skipif(not torch.cuda.is_available(), reason='test requires cuda') def test_ssim_half_gpu(self, preds, target, multichannel): - self.run_precision_test_gpu( preds, target, SSIM, ssim, {"data_range": 1.0}) + self.run_precision_test_gpu(preds, target, SSIM, ssim, {"data_range": 1.0}) @pytest.mark.parametrize( diff --git a/torchmetrics/functional/regression/explained_variance.py b/torchmetrics/functional/regression/explained_variance.py index 0f38b2259ab..8abcaea93a8 100644 --- a/torchmetrics/functional/regression/explained_variance.py +++ b/torchmetrics/functional/regression/explained_variance.py @@ -24,10 +24,11 @@ def _explained_variance_update(preds: Tensor, target: Tensor) -> Tuple[int, Tens n_obs = preds.size(0) sum_error = torch.sum(target - preds, dim=0) - sum_squared_error = torch.sum((target - preds)**2, dim=0) + diff = target - preds + sum_squared_error = torch.sum(diff*diff, dim=0) sum_target = torch.sum(target, dim=0) - sum_squared_target = torch.sum(target**2, dim=0) + sum_squared_target = torch.sum(target*target, dim=0) return n_obs, sum_error, sum_squared_error, sum_target, sum_squared_target @@ -41,10 +42,10 @@ def _explained_variance_compute( multioutput: str = "uniform_average", ) -> Union[Tensor, Sequence[Tensor]]: diff_avg = sum_error / n_obs - numerator = sum_squared_error / n_obs - diff_avg**2 + numerator = sum_squared_error / n_obs - (diff_avg*diff_avg) target_avg = sum_target / n_obs - denominator = sum_squared_target / n_obs - target_avg**2 + denominator = sum_squared_target / n_obs - (target_avg*target_avg) # Take care of division by zero nonzero_numerator = numerator != 0 From 5c82c97c8bf754a0bf0377f5bdc6369b920c76bf Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Fri, 26 Mar 2021 12:02:07 +0100 Subject: [PATCH 3/5] r2 --- tests/helpers/testers.py | 4 ++-- tests/regression/test_explained_variance.py | 2 +- tests/regression/test_r2score.py | 9 +++++++++ tests/regression/test_ssim.py | 2 +- torchmetrics/functional/regression/explained_variance.py | 8 ++++---- torchmetrics/functional/regression/r2score.py | 5 +++-- 6 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/helpers/testers.py b/tests/helpers/testers.py index e534bc7fc96..80aa855e3dd 100644 --- a/tests/helpers/testers.py +++ b/tests/helpers/testers.py @@ -190,8 +190,8 @@ def _assert_half_support( y_hat = preds[0].half().to(device) if preds[0].is_floating_point() else preds[0].to(device) y = target[0].half().to(device) if target[0].is_floating_point() else target[0].to(device) metric_module = metric_module.to(device) - assert metric_module(y_hat, y) - assert metric_functional(y_hat, y) + _assert_tensor(metric_module(y_hat, y)) + _assert_tensor(metric_functional(y_hat, y)) class MetricTester: diff --git a/tests/regression/test_explained_variance.py b/tests/regression/test_explained_variance.py index 79d1aafa993..0e458926215 100644 --- a/tests/regression/test_explained_variance.py +++ b/tests/regression/test_explained_variance.py @@ -83,7 +83,7 @@ def test_explained_variance_functional(self, multioutput, preds, target, sk_metr partial(sk_metric, sk_fn=partial(explained_variance_score, multioutput=multioutput)), metric_args=dict(multioutput=multioutput), ) - + def test_explained_variance_half_cpu(self, multioutput, preds, target, sk_metric): self.run_precision_test_cpu(preds, target, ExplainedVariance, explained_variance) diff --git a/tests/regression/test_r2score.py b/tests/regression/test_r2score.py index 3d0281244ec..91d1c3b996c 100644 --- a/tests/regression/test_r2score.py +++ b/tests/regression/test_r2score.py @@ -92,6 +92,15 @@ def test_r2_functional(self, adjusted, multioutput, preds, target, sk_metric, nu metric_args=dict(adjusted=adjusted, multioutput=multioutput), ) + def test_r2_half_cpu(self, adjusted, multioutput, preds, target, sk_metric, num_outputs): + self.run_precision_test_cpu(preds, target, partial(R2Score, num_outputs=num_outputs), r2score, + {'adjusted': adjusted, 'multioutput': multioutput}) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason='test requires cuda') + def test_r2_half_gpu(self, adjusted, multioutput, preds, target, sk_metric, num_outputs): + self.run_precision_test_gpu(preds, target, partial(R2Score, num_outputs=num_outputs), r2score, + {'adjusted': adjusted, 'multioutput': multioutput}) + def test_error_on_different_shape(metric_class=R2Score): metric = metric_class() diff --git a/tests/regression/test_ssim.py b/tests/regression/test_ssim.py index b6ec163db46..cea7ecb6d0f 100644 --- a/tests/regression/test_ssim.py +++ b/tests/regression/test_ssim.py @@ -90,7 +90,7 @@ def test_ssim_functional(self, preds, target, multichannel): partial(_sk_metric, data_range=1.0, multichannel=multichannel), metric_args={"data_range": 1.0}, ) - + # SSIM half + cpu does not work due to missing support in torch.log @pytest.mark.xfail(reason="SSIM metric does not support cpu + half precision") def test_ssim_half_cpu(self, preds, target, multichannel): diff --git a/torchmetrics/functional/regression/explained_variance.py b/torchmetrics/functional/regression/explained_variance.py index 8abcaea93a8..a373c8687da 100644 --- a/torchmetrics/functional/regression/explained_variance.py +++ b/torchmetrics/functional/regression/explained_variance.py @@ -25,10 +25,10 @@ def _explained_variance_update(preds: Tensor, target: Tensor) -> Tuple[int, Tens n_obs = preds.size(0) sum_error = torch.sum(target - preds, dim=0) diff = target - preds - sum_squared_error = torch.sum(diff*diff, dim=0) + sum_squared_error = torch.sum(diff * diff, dim=0) sum_target = torch.sum(target, dim=0) - sum_squared_target = torch.sum(target*target, dim=0) + sum_squared_target = torch.sum(target * target, dim=0) return n_obs, sum_error, sum_squared_error, sum_target, sum_squared_target @@ -42,10 +42,10 @@ def _explained_variance_compute( multioutput: str = "uniform_average", ) -> Union[Tensor, Sequence[Tensor]]: diff_avg = sum_error / n_obs - numerator = sum_squared_error / n_obs - (diff_avg*diff_avg) + numerator = sum_squared_error / n_obs - (diff_avg * diff_avg) target_avg = sum_target / n_obs - denominator = sum_squared_target / n_obs - (target_avg*target_avg) + denominator = sum_squared_target / n_obs - (target_avg * target_avg) # Take care of division by zero nonzero_numerator = numerator != 0 diff --git a/torchmetrics/functional/regression/r2score.py b/torchmetrics/functional/regression/r2score.py index ebafa94b61a..ae9d68b58e8 100644 --- a/torchmetrics/functional/regression/r2score.py +++ b/torchmetrics/functional/regression/r2score.py @@ -31,8 +31,9 @@ def _r2score_update(preds: Tensor, target: Tensor) -> Tuple[Tensor, Tensor, Tens raise ValueError('Needs at least two samples to calculate r2 score.') sum_error = torch.sum(target, dim=0) - sum_squared_error = torch.sum(torch.pow(target, 2.0), dim=0) - residual = torch.sum(torch.pow(target - preds, 2.0), dim=0) + sum_squared_error = torch.sum(target * target, dim=0) + diff = target - preds + residual = torch.sum(diff * diff, dim=0) total = target.size(0) return sum_squared_error, sum_error, residual, total From ff0fccc8611d581bc849a3925f354cb09494d41e Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Fri, 26 Mar 2021 12:09:46 +0100 Subject: [PATCH 4/5] changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05efd88d929..97aae384402 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added multilabel support to `ROC` metric ([#114](https://github.com/PyTorchLightning/metrics/pull/114)) -- Added testing for `half` precision ([#77](https://github.com/PyTorchLightning/metrics/pull/77)) +- Added testing for `half` precision ([#77](https://github.com/PyTorchLightning/metrics/pull/77), + [#135](https://github.com/PyTorchLightning/metrics/pull/135) +) ### Changed From 9ac2e4b521cff6ff7bd461aa2fe0ac4283e92b1c Mon Sep 17 00:00:00 2001 From: Nicki Skafte Date: Fri, 26 Mar 2021 12:57:55 +0100 Subject: [PATCH 5/5] minimum --- tests/regression/test_explained_variance.py | 4 ++++ tests/regression/test_r2score.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/regression/test_explained_variance.py b/tests/regression/test_explained_variance.py index 0e458926215..dd1c37a4969 100644 --- a/tests/regression/test_explained_variance.py +++ b/tests/regression/test_explained_variance.py @@ -22,6 +22,7 @@ from tests.helpers.testers import BATCH_SIZE, NUM_BATCHES, MetricTester from torchmetrics.functional import explained_variance from torchmetrics.regression import ExplainedVariance +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_6 seed_all(42) @@ -84,6 +85,9 @@ def test_explained_variance_functional(self, multioutput, preds, target, sk_metr metric_args=dict(multioutput=multioutput), ) + @pytest.mark.skipif( + not _TORCH_GREATER_EQUAL_1_6, reason='half support of core operations on not support before pytorch v1.6' + ) def test_explained_variance_half_cpu(self, multioutput, preds, target, sk_metric): self.run_precision_test_cpu(preds, target, ExplainedVariance, explained_variance) diff --git a/tests/regression/test_r2score.py b/tests/regression/test_r2score.py index 91d1c3b996c..aee3ecbb946 100644 --- a/tests/regression/test_r2score.py +++ b/tests/regression/test_r2score.py @@ -22,6 +22,7 @@ from tests.helpers.testers import BATCH_SIZE, NUM_BATCHES, MetricTester from torchmetrics.functional import r2score from torchmetrics.regression import R2Score +from torchmetrics.utilities.imports import _TORCH_GREATER_EQUAL_1_6 seed_all(42) @@ -92,6 +93,9 @@ def test_r2_functional(self, adjusted, multioutput, preds, target, sk_metric, nu metric_args=dict(adjusted=adjusted, multioutput=multioutput), ) + @pytest.mark.skipif( + not _TORCH_GREATER_EQUAL_1_6, reason='half support of core operations on not support before pytorch v1.6' + ) def test_r2_half_cpu(self, adjusted, multioutput, preds, target, sk_metric, num_outputs): self.run_precision_test_cpu(preds, target, partial(R2Score, num_outputs=num_outputs), r2score, {'adjusted': adjusted, 'multioutput': multioutput})