From ca6dcb75c5e8cb39589f49415eb582e45bc24d67 Mon Sep 17 00:00:00 2001 From: MapleFengLiang Date: Sun, 14 Feb 2021 20:33:16 +0300 Subject: [PATCH 01/17] added ipython env check --- .../accelerators/accelerator_connector.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/accelerators/accelerator_connector.py b/pytorch_lightning/accelerators/accelerator_connector.py index cfa9545ad6aee..412d72c7d9af3 100644 --- a/pytorch_lightning/accelerators/accelerator_connector.py +++ b/pytorch_lightning/accelerators/accelerator_connector.py @@ -500,7 +500,28 @@ def set_distributed_mode(self, distributed_backend: Optional[str] = None): else: rank_zero_warn('You are running on single node with no parallelization, so distributed has no effect.') self._distrib_type = None - + + # finished configuring self._distrib_type, check ipython environment + ipython_incompatible_distrib_types = [DistributedType.DDP, DistributedType.DDP2, DistributedType.HOROVOD] + if self._distrib_type in ipython_incompatible_distrib_types: + # check ipython env + import sys + in_ipython = False + in_ipython_kernel = False + if 'IPython' in sys.modules: + from IPython import get_ipython + ip = get_ipython() + in_ipython = ip is not None + + if in_ipython: + in_ipython_kernel = getattr(ip, 'kernel', None) is not None + + if in_ipython_kernel: + raise MisconfigurationException( + "Selected distributed backend {self._distrib_type} not compatible with IPython environment" + "Run your code as a script, or choose DDP_Spawn/DP as accelerator backend" + ) + # for DDP overwrite nb processes by requested GPUs if ( self._device_type == DeviceType.GPU From df6348d491fffc26643ae3b071a857e4d0e3140c Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Mon, 15 Feb 2021 18:32:01 +0300 Subject: [PATCH 02/17] extracted a method and added a compatibility list --- .../accelerators/accelerator_connector.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator_connector.py b/pytorch_lightning/accelerators/accelerator_connector.py index 412d72c7d9af3..e7a6e39e885d5 100644 --- a/pytorch_lightning/accelerators/accelerator_connector.py +++ b/pytorch_lightning/accelerators/accelerator_connector.py @@ -82,6 +82,7 @@ def __init__( # initialization self._device_type = DeviceType.CPU self._distrib_type = None + self.ipython_compatible_distrib_types = [DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN] self.num_processes = num_processes self.tpu_cores = device_parser.parse_tpu_cores(tpu_cores) @@ -502,25 +503,7 @@ def set_distributed_mode(self, distributed_backend: Optional[str] = None): self._distrib_type = None # finished configuring self._distrib_type, check ipython environment - ipython_incompatible_distrib_types = [DistributedType.DDP, DistributedType.DDP2, DistributedType.HOROVOD] - if self._distrib_type in ipython_incompatible_distrib_types: - # check ipython env - import sys - in_ipython = False - in_ipython_kernel = False - if 'IPython' in sys.modules: - from IPython import get_ipython - ip = get_ipython() - in_ipython = ip is not None - - if in_ipython: - in_ipython_kernel = getattr(ip, 'kernel', None) is not None - - if in_ipython_kernel: - raise MisconfigurationException( - "Selected distributed backend {self._distrib_type} not compatible with IPython environment" - "Run your code as a script, or choose DDP_Spawn/DP as accelerator backend" - ) + self.check_ipython_compatibility() # for DDP overwrite nb processes by requested GPUs if ( @@ -563,6 +546,27 @@ def _set_horovod_backend(self): else: self.num_processes = hvd.local_size() + def check_ipython_compatibility(self): + """Raises a `MisconfigurationException` if the accelerator is not compatible with IPython and code is run in an IPython kernel.""" + if self._distrib_type not in self.ipython_compatible_distrib_types: + # check ipython env + import sys + in_ipython = False + in_ipython_kernel = False + if 'IPython' in sys.modules: + from IPython import get_ipython + ip = get_ipython() + in_ipython = ip is not None + + if in_ipython: + in_ipython_kernel = getattr(ip, 'kernel', None) is not None + + if in_ipython_kernel: + raise MisconfigurationException( + "Selected distributed backend {self._distrib_type} not compatible with IPython environment" + "Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" + ) + def check_horovod(self): """Raises a `MisconfigurationException` if the Trainer is not configured correctly for Horovod.""" if not _HOROVOD_AVAILABLE: From 5ad468ebb3ddb0710c48cb0108e2b710d3faaa29 Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Tue, 16 Feb 2021 21:51:36 +0300 Subject: [PATCH 03/17] fixed fstring --- .../accelerators/accelerator_connector.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator_connector.py b/pytorch_lightning/accelerators/accelerator_connector.py index e7a6e39e885d5..7409a5f8ada35 100644 --- a/pytorch_lightning/accelerators/accelerator_connector.py +++ b/pytorch_lightning/accelerators/accelerator_connector.py @@ -82,7 +82,9 @@ def __init__( # initialization self._device_type = DeviceType.CPU self._distrib_type = None - self.ipython_compatible_distrib_types = [DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN] + self.ipython_compatible_distrib_types = [ + DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN + ] self.num_processes = num_processes self.tpu_cores = device_parser.parse_tpu_cores(tpu_cores) @@ -501,10 +503,10 @@ def set_distributed_mode(self, distributed_backend: Optional[str] = None): else: rank_zero_warn('You are running on single node with no parallelization, so distributed has no effect.') self._distrib_type = None - + # finished configuring self._distrib_type, check ipython environment self.check_ipython_compatibility() - + # for DDP overwrite nb processes by requested GPUs if ( self._device_type == DeviceType.GPU @@ -560,12 +562,12 @@ def check_ipython_compatibility(self): if in_ipython: in_ipython_kernel = getattr(ip, 'kernel', None) is not None - + if in_ipython_kernel: raise MisconfigurationException( - "Selected distributed backend {self._distrib_type} not compatible with IPython environment" - "Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" - ) + f"Selected distributed backend {self._distrib_type} not compatible with IPython environment" + f"Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" + ) def check_horovod(self): """Raises a `MisconfigurationException` if the Trainer is not configured correctly for Horovod.""" From 46dcd8df8ec5377bf875bd03ed84e648669cee22 Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Tue, 16 Feb 2021 22:11:22 +0300 Subject: [PATCH 04/17] inverted if, added early return --- .../accelerators/accelerator_connector.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator_connector.py b/pytorch_lightning/accelerators/accelerator_connector.py index 7409a5f8ada35..c001b1eb391fb 100644 --- a/pytorch_lightning/accelerators/accelerator_connector.py +++ b/pytorch_lightning/accelerators/accelerator_connector.py @@ -550,24 +550,26 @@ def _set_horovod_backend(self): def check_ipython_compatibility(self): """Raises a `MisconfigurationException` if the accelerator is not compatible with IPython and code is run in an IPython kernel.""" - if self._distrib_type not in self.ipython_compatible_distrib_types: + if self._distrib_type in self.ipython_compatible_distrib_types: + return + else: # check ipython env import sys - in_ipython = False - in_ipython_kernel = False if 'IPython' in sys.modules: from IPython import get_ipython + in_ipython = False + in_ipython_kernel = False ip = get_ipython() in_ipython = ip is not None - if in_ipython: - in_ipython_kernel = getattr(ip, 'kernel', None) is not None + if in_ipython: + in_ipython_kernel = getattr(ip, 'kernel', None) is not None - if in_ipython_kernel: - raise MisconfigurationException( - f"Selected distributed backend {self._distrib_type} not compatible with IPython environment" - f"Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" - ) + if in_ipython_kernel: + raise MisconfigurationException( + f"Selected distributed backend {self._distrib_type} not compatible with IPython environment" + f"Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" + ) def check_horovod(self): """Raises a `MisconfigurationException` if the Trainer is not configured correctly for Horovod.""" From 6739c3681a7ad1c780a99c8c388db5b6c4cec218 Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Wed, 17 Feb 2021 21:14:05 +0300 Subject: [PATCH 05/17] changed to only checking ipython env --- .../accelerators/accelerator_connector.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/pytorch_lightning/accelerators/accelerator_connector.py b/pytorch_lightning/accelerators/accelerator_connector.py index c001b1eb391fb..347455e253f4d 100644 --- a/pytorch_lightning/accelerators/accelerator_connector.py +++ b/pytorch_lightning/accelerators/accelerator_connector.py @@ -557,15 +557,7 @@ def check_ipython_compatibility(self): import sys if 'IPython' in sys.modules: from IPython import get_ipython - in_ipython = False - in_ipython_kernel = False - ip = get_ipython() - in_ipython = ip is not None - - if in_ipython: - in_ipython_kernel = getattr(ip, 'kernel', None) is not None - - if in_ipython_kernel: + if get_ipython() is not None: raise MisconfigurationException( f"Selected distributed backend {self._distrib_type} not compatible with IPython environment" f"Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" From da9988697d659a5c5952a194375e74dadfa8beec Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Thu, 18 Feb 2021 15:45:34 +0300 Subject: [PATCH 06/17] move import to top --- pytorch_lightning/accelerators/accelerator_connector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/accelerators/accelerator_connector.py b/pytorch_lightning/accelerators/accelerator_connector.py index 347455e253f4d..37f5cee2bd8d1 100644 --- a/pytorch_lightning/accelerators/accelerator_connector.py +++ b/pytorch_lightning/accelerators/accelerator_connector.py @@ -13,6 +13,8 @@ # limitations under the License. import os +import sys + from typing import List, Optional, Sequence, Union import torch @@ -554,7 +556,6 @@ def check_ipython_compatibility(self): return else: # check ipython env - import sys if 'IPython' in sys.modules: from IPython import get_ipython if get_ipython() is not None: From a6eaec240cdcc8266ceba4b3aba79cf191ead9dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 19 Feb 2021 00:08:02 +0100 Subject: [PATCH 07/17] fix formatting, docstring and line length --- .../connectors/accelerator_connector.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 70f6d404e8d41..25c6e5a1ea915 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -565,18 +565,21 @@ def _set_horovod_backend(self): self.num_processes = hvd.local_size() def check_ipython_compatibility(self): - """Raises a `MisconfigurationException` if the accelerator is not compatible with IPython and code is run in an IPython kernel.""" + """ + Raises a `MisconfigurationException` if the accelerator and/or plugin is not compatible with IPython + and code is run in an IPython kernel. + """ if self._distrib_type in self.ipython_compatible_distrib_types: return - else: - # check ipython env - if 'IPython' in sys.modules: - from IPython import get_ipython - if get_ipython() is not None: - raise MisconfigurationException( - f"Selected distributed backend {self._distrib_type} not compatible with IPython environment" - f"Run your code as a script, or choose one of compatible backends {self.ipython_compatible_distrib_types} as accelerator backend" - ) + # check ipython env + if "IPython" in sys.modules: + from IPython import get_ipython + if get_ipython() is not None: + raise MisconfigurationException( + f"Selected distributed backend {self._distrib_type} is not compatible with IPython environment." + f" Run your code as a script, or choose one of the compatible backends:" + f" {', '.join(self.ipython_compatible_distrib_types)}" + ) def check_horovod(self): """Raises a `MisconfigurationException` if the Trainer is not configured correctly for Horovod.""" From c0ff8ce080fca61a581df3ad737daac94859c3c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 19 Feb 2021 00:08:43 +0100 Subject: [PATCH 08/17] fix isort --- pytorch_lightning/trainer/connectors/accelerator_connector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 9d29d8b17875a..f098658a25bd6 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -14,7 +14,6 @@ import os import sys - from typing import List, Optional, Sequence, Union import torch From 70b7dbf3146f59dacde271379fa47c4754245b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Fri, 19 Feb 2021 00:51:20 +0100 Subject: [PATCH 09/17] add test --- .../accelerators/test_accelerator_connector.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/accelerators/test_accelerator_connector.py b/tests/accelerators/test_accelerator_connector.py index 76d4a597d8ecb..238a96cdccbf3 100644 --- a/tests/accelerators/test_accelerator_connector.py +++ b/tests/accelerators/test_accelerator_connector.py @@ -13,7 +13,9 @@ # limitations under the License import os +import sys from unittest import mock +from unittest.mock import Mock import pytest import torch @@ -25,6 +27,7 @@ from pytorch_lightning.callbacks import Callback from pytorch_lightning.plugins import DDP2Plugin, DDPPlugin, DDPSpawnPlugin, PrecisionPlugin, SingleDevicePlugin from pytorch_lightning.plugins.environments import ClusterEnvironment, SLURMEnvironment, TorchElasticEnvironment +from pytorch_lightning.utilities.exceptions import MisconfigurationException from tests.helpers.boring_model import BoringModel @@ -378,3 +381,17 @@ def on_fit_start(self, trainer, pl_module): with pytest.raises(SystemExit): trainer.fit(model) + + +@mock.patch.dict(sys.modules, {"IPython": Mock()}) +@mock.patch("IPython.get_ipython") +@mock.patch('torch.cuda.device_count', return_value=2) +def test_ipython_incompatible_backend_error(device_count_mock, ipython_mock): + with pytest.raises(MisconfigurationException, match="backend ddp is not compatible"): + Trainer(accelerator="ddp", gpus=2) + + with pytest.raises(MisconfigurationException, match="backend ddp is not compatible"): + Trainer(accelerator="ddp_cpu", num_processes=2) + + with pytest.raises(MisconfigurationException, match="backend ddp2 is not compatible"): + Trainer(accelerator="ddp2", gpus=2) From 53dc217dd08336ee6e0b2163bf6850b006783604 Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Fri, 19 Feb 2021 16:59:11 +0300 Subject: [PATCH 10/17] moved compatible list to enum method --- .../trainer/connectors/accelerator_connector.py | 5 +---- pytorch_lightning/utilities/enums.py | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 70f6d404e8d41..8713f9960e78d 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -86,9 +86,6 @@ def __init__( # initialization self._device_type = DeviceType.CPU self._distrib_type = None - self.ipython_compatible_distrib_types = [ - DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN - ] self.num_processes = num_processes self.tpu_cores = device_parser.parse_tpu_cores(tpu_cores) @@ -566,7 +563,7 @@ def _set_horovod_backend(self): def check_ipython_compatibility(self): """Raises a `MisconfigurationException` if the accelerator is not compatible with IPython and code is run in an IPython kernel.""" - if self._distrib_type in self.ipython_compatible_distrib_types: + if self._distrib_type in DistributedType.ipython_compatible_types(): return else: # check ipython env diff --git a/pytorch_lightning/utilities/enums.py b/pytorch_lightning/utilities/enums.py index 3e4add4fb68d1..f3d3e03212aaa 100644 --- a/pytorch_lightning/utilities/enums.py +++ b/pytorch_lightning/utilities/enums.py @@ -58,6 +58,12 @@ class DistributedType(LightningEnum): >>> DistributedType.DDP2 in ('ddp2', ) True """ + + @staticmethod + def ipython_compatible_types() -> list: + """Returns a list containing ipython compatible DistributeTypes""" + return [DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN] + DP = 'dp' DDP = 'ddp' DDP2 = 'ddp2' From 0676368bc615f53c80024862290ff15dfce17907 Mon Sep 17 00:00:00 2001 From: ifsheldon Date: Fri, 19 Feb 2021 17:12:51 +0300 Subject: [PATCH 11/17] fixed a minor print issue --- pytorch_lightning/trainer/connectors/accelerator_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 059bd5719f9c9..69a8a20976100 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -574,7 +574,7 @@ def check_ipython_compatibility(self): raise MisconfigurationException( f"Selected distributed backend {self._distrib_type} is not compatible with IPython environment." f" Run your code as a script, or choose one of the compatible backends:" - f" {', '.join(self.ipython_compatible_distrib_types)}" + f" {', '.join(DistributedType.ipython_compatible_types())}" ) def check_horovod(self): From 386e3a9e5132064761bb6c99f793125a01ae33f7 Mon Sep 17 00:00:00 2001 From: MapleFengLiang Date: Fri, 19 Feb 2021 17:45:31 +0300 Subject: [PATCH 12/17] changed to use utilities.imports --- pytorch_lightning/trainer/connectors/accelerator_connector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 69a8a20976100..5805751d910f9 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import sys from typing import List, Optional, Sequence, Union import torch @@ -55,6 +54,7 @@ DeviceType, DistributedType, rank_zero_only, + _module_available, ) from pytorch_lightning.utilities.distributed import rank_zero_info, rank_zero_warn from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -568,7 +568,7 @@ def check_ipython_compatibility(self): if self._distrib_type in DistributedType.ipython_compatible_types(): return # check ipython env - if "IPython" in sys.modules: + if _module_available("IPython"): from IPython import get_ipython if get_ipython() is not None: raise MisconfigurationException( From 4384880d95bbf5f9d3057c9b589908d1646ad38b Mon Sep 17 00:00:00 2001 From: MapleFengLiang Date: Fri, 19 Feb 2021 17:48:26 +0300 Subject: [PATCH 13/17] added a method to check ipython compatibility --- pytorch_lightning/trainer/connectors/accelerator_connector.py | 2 +- pytorch_lightning/utilities/enums.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 5805751d910f9..38bc874d86cff 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -565,7 +565,7 @@ def check_ipython_compatibility(self): Raises a `MisconfigurationException` if the accelerator and/or plugin is not compatible with IPython and code is run in an IPython kernel. """ - if self._distrib_type in DistributedType.ipython_compatible_types(): + if self._distrib_type.is_ipython_compatible(): return # check ipython env if _module_available("IPython"): diff --git a/pytorch_lightning/utilities/enums.py b/pytorch_lightning/utilities/enums.py index f3d3e03212aaa..8bab90ab44090 100644 --- a/pytorch_lightning/utilities/enums.py +++ b/pytorch_lightning/utilities/enums.py @@ -64,6 +64,10 @@ def ipython_compatible_types() -> list: """Returns a list containing ipython compatible DistributeTypes""" return [DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN] + def is_ipython_compatible(self) -> bool: + """Returns whether self is ipython compatible""" + return self in DistributedType.ipython_compatible_types() + DP = 'dp' DDP = 'ddp' DDP2 = 'ddp2' From 73b18b141d1ab3081ad8315ae0371e08089eb312 Mon Sep 17 00:00:00 2001 From: MapleFengLiang Date: Fri, 19 Feb 2021 18:03:46 +0300 Subject: [PATCH 14/17] fixed a minor issue when _distrib_type is None --- pytorch_lightning/trainer/connectors/accelerator_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 38bc874d86cff..5d4232dbb86db 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -565,7 +565,7 @@ def check_ipython_compatibility(self): Raises a `MisconfigurationException` if the accelerator and/or plugin is not compatible with IPython and code is run in an IPython kernel. """ - if self._distrib_type.is_ipython_compatible(): + if self._distrib_type is None or self._distrib_type.is_ipython_compatible(): return # check ipython env if _module_available("IPython"): From d2dedab6bb54df49a417d9e086bfd4dfda805caf Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Mon, 22 Feb 2021 01:25:11 +0100 Subject: [PATCH 15/17] Fix test --- pytorch_lightning/utilities/enums.py | 8 ++++---- tests/accelerators/test_accelerator_connector.py | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/utilities/enums.py b/pytorch_lightning/utilities/enums.py index 8bab90ab44090..38d4f5e15ad29 100644 --- a/pytorch_lightning/utilities/enums.py +++ b/pytorch_lightning/utilities/enums.py @@ -13,14 +13,14 @@ # limitations under the License. """Enumerated utilities""" from enum import Enum -from typing import Union +from typing import List, Optional, Union class LightningEnum(str, Enum): """ Type of any enumerator with allowed comparison to string invariant to cases. """ @classmethod - def from_str(cls, value: str) -> 'LightningEnum': + def from_str(cls, value: str) -> Optional['LightningEnum']: statuses = [status for status in dir(cls) if not status.startswith('_')] for st in statuses: if st.lower() == value.lower(): @@ -31,7 +31,7 @@ def __eq__(self, other: Union[str, Enum]) -> bool: other = other.value if isinstance(other, Enum) else str(other) return self.value.lower() == other.lower() - def __hash__(self): + def __hash__(self) -> int: # re-enable hashtable so it can be used as a dict key or in a set # example: set(LightningEnum) return hash(self.name) @@ -60,7 +60,7 @@ class DistributedType(LightningEnum): """ @staticmethod - def ipython_compatible_types() -> list: + def ipython_compatible_types() -> List['DistributedType']: """Returns a list containing ipython compatible DistributeTypes""" return [DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN] diff --git a/tests/accelerators/test_accelerator_connector.py b/tests/accelerators/test_accelerator_connector.py index 238a96cdccbf3..e054525447b16 100644 --- a/tests/accelerators/test_accelerator_connector.py +++ b/tests/accelerators/test_accelerator_connector.py @@ -383,10 +383,9 @@ def on_fit_start(self, trainer, pl_module): trainer.fit(model) -@mock.patch.dict(sys.modules, {"IPython": Mock()}) @mock.patch("IPython.get_ipython") @mock.patch('torch.cuda.device_count', return_value=2) -def test_ipython_incompatible_backend_error(device_count_mock, ipython_mock): +def test_ipython_incompatible_backend_error(*_): with pytest.raises(MisconfigurationException, match="backend ddp is not compatible"): Trainer(accelerator="ddp", gpus=2) From 990c03ef6ecafb4c3a82a85bb54ac89c6d3bfcb4 Mon Sep 17 00:00:00 2001 From: Carlos Mocholi Date: Tue, 23 Feb 2021 02:40:49 +0100 Subject: [PATCH 16/17] IPython -> interactive --- .../connectors/accelerator_connector.py | 27 ++++++++----------- pytorch_lightning/utilities/__init__.py | 2 +- pytorch_lightning/utilities/enums.py | 10 +++---- pytorch_lightning/utilities/imports.py | 5 +++- .../test_accelerator_connector.py | 3 +-- 5 files changed, 22 insertions(+), 25 deletions(-) diff --git a/pytorch_lightning/trainer/connectors/accelerator_connector.py b/pytorch_lightning/trainer/connectors/accelerator_connector.py index 55ad65790e150..1b2eed56b84bc 100644 --- a/pytorch_lightning/trainer/connectors/accelerator_connector.py +++ b/pytorch_lightning/trainer/connectors/accelerator_connector.py @@ -54,7 +54,6 @@ DeviceType, DistributedType, rank_zero_only, - _module_available, ) from pytorch_lightning.utilities.distributed import rank_zero_info, rank_zero_warn from pytorch_lightning.utilities.exceptions import MisconfigurationException @@ -519,7 +518,7 @@ def set_distributed_mode(self, distributed_backend: Optional[str] = None): self._distrib_type = None # finished configuring self._distrib_type, check ipython environment - self.check_ipython_compatibility() + self.check_interactive_compatibility() # for DDP overwrite nb processes by requested GPUs if ( @@ -562,22 +561,18 @@ def _set_horovod_backend(self): else: self.num_processes = hvd.local_size() - def check_ipython_compatibility(self): + def check_interactive_compatibility(self): """ - Raises a `MisconfigurationException` if the accelerator and/or plugin is not compatible with IPython - and code is run in an IPython kernel. + Raises a `MisconfigurationException` if the accelerator and/or plugin + is not compatible with an interactive environment """ - if self._distrib_type is None or self._distrib_type.is_ipython_compatible(): - return - # check ipython env - if _module_available("IPython"): - from IPython import get_ipython - if get_ipython() is not None: - raise MisconfigurationException( - f"Selected distributed backend {self._distrib_type} is not compatible with IPython environment." - f" Run your code as a script, or choose one of the compatible backends:" - f" {', '.join(DistributedType.ipython_compatible_types())}" - ) + from pytorch_lightning.utilities import _IS_INTERACTIVE + if _IS_INTERACTIVE and self._distrib_type is not None and not self._distrib_type.is_interactive_compatible(): + raise MisconfigurationException( + f"Selected distributed backend {self._distrib_type} is not compatible with an interactive" + " environment. Run your code as a script, or choose one of the compatible backends:" + f" {', '.join(DistributedType.interactive_compatible_types())}" + ) def check_horovod(self): """Raises a `MisconfigurationException` if the Trainer is not configured correctly for Horovod.""" diff --git a/pytorch_lightning/utilities/__init__.py b/pytorch_lightning/utilities/__init__.py index cf3aa06f305b8..3e2ee3e51efe1 100644 --- a/pytorch_lightning/utilities/__init__.py +++ b/pytorch_lightning/utilities/__init__.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """General utilities""" - import numpy from pytorch_lightning.utilities.apply_func import move_data_to_device # noqa: F401 @@ -33,6 +32,7 @@ _HOROVOD_AVAILABLE, _HYDRA_AVAILABLE, _HYDRA_EXPERIMENTAL_AVAILABLE, + _IS_INTERACTIVE, _module_available, _NATIVE_AMP_AVAILABLE, _OMEGACONF_AVAILABLE, diff --git a/pytorch_lightning/utilities/enums.py b/pytorch_lightning/utilities/enums.py index 38d4f5e15ad29..169481fa63e67 100644 --- a/pytorch_lightning/utilities/enums.py +++ b/pytorch_lightning/utilities/enums.py @@ -60,13 +60,13 @@ class DistributedType(LightningEnum): """ @staticmethod - def ipython_compatible_types() -> List['DistributedType']: - """Returns a list containing ipython compatible DistributeTypes""" + def interactive_compatible_types() -> List['DistributedType']: + """Returns a list containing interactive compatible DistributeTypes""" return [DistributedType.DP, DistributedType.DDP_SPAWN, DistributedType.DDP_SHARDED_SPAWN] - def is_ipython_compatible(self) -> bool: - """Returns whether self is ipython compatible""" - return self in DistributedType.ipython_compatible_types() + def is_interactive_compatible(self) -> bool: + """Returns whether self is interactive compatible""" + return self in DistributedType.interactive_compatible_types() DP = 'dp' DDP = 'ddp' diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 8024997382457..41a13d6c678a0 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -14,6 +14,7 @@ """General utilities""" import operator import platform +import sys from distutils.version import LooseVersion from importlib.util import find_spec @@ -49,10 +50,11 @@ def _compare_version(package: str, op, version) -> bool: _IS_WINDOWS = platform.system() == "Windows" +_IS_INTERACTIVE = hasattr(sys, "ps1") # https://stackoverflow.com/a/64523765 _TORCH_LOWER_EQUAL_1_4 = _compare_version("torch", operator.le, "1.5.0") _TORCH_GREATER_EQUAL_1_6 = _compare_version("torch", operator.ge, "1.6.0") _TORCH_GREATER_EQUAL_1_7 = _compare_version("torch", operator.ge, "1.7.0") -_TORCH_QUANTIZE_AVAILABLE = bool([eg for eg in torch.backends.quantized.supported_engines if eg != 'none']) + _APEX_AVAILABLE = _module_available("apex.amp") _BOLTS_AVAILABLE = _module_available('pl_bolts') _DEEPSPEED_AVAILABLE = not _IS_WINDOWS and _module_available('deepspeed') @@ -65,6 +67,7 @@ def _compare_version(package: str, op, version) -> bool: _NATIVE_AMP_AVAILABLE = _module_available("torch.cuda.amp") and hasattr(torch.cuda.amp, "autocast") _OMEGACONF_AVAILABLE = _module_available("omegaconf") _RPC_AVAILABLE = not _IS_WINDOWS and _module_available('torch.distributed.rpc') +_TORCH_QUANTIZE_AVAILABLE = bool([eg for eg in torch.backends.quantized.supported_engines if eg != 'none']) _TORCHTEXT_AVAILABLE = _module_available("torchtext") _TORCHVISION_AVAILABLE = _module_available('torchvision') _XLA_AVAILABLE = _module_available("torch_xla") diff --git a/tests/accelerators/test_accelerator_connector.py b/tests/accelerators/test_accelerator_connector.py index eb112c2d006e4..03c5f915343a5 100644 --- a/tests/accelerators/test_accelerator_connector.py +++ b/tests/accelerators/test_accelerator_connector.py @@ -15,7 +15,6 @@ import os import sys from unittest import mock -from unittest.mock import Mock import pytest import torch @@ -390,7 +389,7 @@ def on_fit_start(self, trainer, pl_module): trainer.fit(model) -@mock.patch("IPython.get_ipython") +@mock.patch("pytorch_lightning.utilities._IS_INTERACTIVE", return_value=True) @mock.patch('torch.cuda.device_count', return_value=2) def test_ipython_incompatible_backend_error(*_): with pytest.raises(MisconfigurationException, match="backend ddp is not compatible"): From 0cbb515c04b8def614ab35e4b819bc207632bb89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mochol=C3=AD?= Date: Tue, 23 Feb 2021 03:30:22 +0100 Subject: [PATCH 17/17] Update tests/accelerators/test_accelerator_connector.py --- tests/accelerators/test_accelerator_connector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/accelerators/test_accelerator_connector.py b/tests/accelerators/test_accelerator_connector.py index 03c5f915343a5..e41c891797cd8 100644 --- a/tests/accelerators/test_accelerator_connector.py +++ b/tests/accelerators/test_accelerator_connector.py @@ -13,7 +13,6 @@ # limitations under the License import os -import sys from unittest import mock import pytest