From 9aad39e936fb51795b954133e25b82f307789191 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 9 Mar 2021 10:29:44 +0100 Subject: [PATCH 01/20] fix comparing versions --- pytorch_lightning/utilities/imports.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 41a13d6c678a0..9e2e38c2698df 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """General utilities""" +import importlib import operator import platform import sys @@ -42,8 +43,15 @@ def _module_available(module_path: str) -> bool: def _compare_version(package: str, op, version) -> bool: + """Compare package version with some requirements + + >>> _compare_version("torch", operator.ge, "0.1") + True + """ try: - pkg_version = LooseVersion(get_distribution(package).version) + pkg = importlib.import_module(package) + assert hasattr(pkg, '__version__') + pkg_version = pkg.__version__ return op(pkg_version, LooseVersion(version)) except DistributionNotFound: return False From b18cfd4ff39587c36b7e4c246b174ecb08aeaeef Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 9 Mar 2021 10:34:29 +0100 Subject: [PATCH 02/20] chlog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51ad97decd867..c542b854af104 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Fixed torch distributed not available in setup hook for DDP ([#6506](https://github.com/PyTorchLightning/pytorch-lightning/pull/6506)) +- Fixed comparing required versions ([#6434](https://github.com/PyTorchLightning/pytorch-lightning/pull/6434)) + + ## [1.2.4] - 2021-03-16 ### Changed From 7b5584c582287e6d24fafac5c773caf5e24058ef Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 9 Mar 2021 10:36:33 +0100 Subject: [PATCH 03/20] . --- pytorch_lightning/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 9e2e38c2698df..70ec05e7e1427 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -20,7 +20,7 @@ from importlib.util import find_spec import torch -from pkg_resources import DistributionNotFound, get_distribution +from pkg_resources import DistributionNotFound def _module_available(module_path: str) -> bool: From f4490864a12a2731f2cab0d0d098dd90a550e9b0 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 9 Mar 2021 11:06:36 +0100 Subject: [PATCH 04/20] ... --- pytorch_lightning/utilities/imports.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 70ec05e7e1427..7bb6f51b1195e 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -48,6 +48,8 @@ def _compare_version(package: str, op, version) -> bool: >>> _compare_version("torch", operator.ge, "0.1") True """ + if not _module_available(package): + return False try: pkg = importlib.import_module(package) assert hasattr(pkg, '__version__') From 77d05af13685d4faf5122398354338e07096c437 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 17 Mar 2021 14:55:58 +0100 Subject: [PATCH 05/20] Apply suggestions from code review --- pl_examples/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pl_examples/__init__.py b/pl_examples/__init__.py index 150ac309ddceb..1459e6f626857 100644 --- a/pl_examples/__init__.py +++ b/pl_examples/__init__.py @@ -18,7 +18,7 @@ _TORCHVISION_MNIST_AVAILABLE = not bool(os.environ.get("PL_USE_MOCKED_MNIST", False)) _DALI_AVAILABLE = _module_available("nvidia.dali") -if _TORCHVISION_MNIST_AVAILABLE: +if _TORCHVISION_AVAILABLE: try: from torchvision.datasets.mnist import MNIST MNIST(_DATASETS_PATH, download=True) From 00c53ca03e78fce405ad1821b39dbdf85a4b494a Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 18:08:03 +0100 Subject: [PATCH 06/20] ... --- pl_examples/__init__.py | 2 +- pytorch_lightning/utilities/imports.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pl_examples/__init__.py b/pl_examples/__init__.py index 1459e6f626857..150ac309ddceb 100644 --- a/pl_examples/__init__.py +++ b/pl_examples/__init__.py @@ -18,7 +18,7 @@ _TORCHVISION_MNIST_AVAILABLE = not bool(os.environ.get("PL_USE_MOCKED_MNIST", False)) _DALI_AVAILABLE = _module_available("nvidia.dali") -if _TORCHVISION_AVAILABLE: +if _TORCHVISION_MNIST_AVAILABLE: try: from torchvision.datasets.mnist import MNIST MNIST(_DATASETS_PATH, download=True) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 7bb6f51b1195e..e07f93b07d221 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -43,7 +43,8 @@ def _module_available(module_path: str) -> bool: def _compare_version(package: str, op, version) -> bool: - """Compare package version with some requirements + """ + Compare package version with some requirements >>> _compare_version("torch", operator.ge, "0.1") True From 52c7e7df8aa1341d7b9a02962f13a7a06c7e22d7 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 21:32:45 +0100 Subject: [PATCH 07/20] .... --- pytorch_lightning/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index e07f93b07d221..7d9fd345a7f95 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -55,7 +55,7 @@ def _compare_version(package: str, op, version) -> bool: pkg = importlib.import_module(package) assert hasattr(pkg, '__version__') pkg_version = pkg.__version__ - return op(pkg_version, LooseVersion(version)) + return op(LooseVersion(pkg_version), LooseVersion(version)) except DistributionNotFound: return False From de2b36ce1dac52901e226a200f69dd234502b5b9 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 21:58:43 +0100 Subject: [PATCH 08/20] -- --- pytorch_lightning/utilities/imports.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 7d9fd345a7f95..53435acef3cec 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -53,11 +53,12 @@ def _compare_version(package: str, op, version) -> bool: return False try: pkg = importlib.import_module(package) - assert hasattr(pkg, '__version__') - pkg_version = pkg.__version__ - return op(LooseVersion(pkg_version), LooseVersion(version)) except DistributionNotFound: return False + if not hasattr(pkg, '__version__'): + # in case version is not defined always return True as likely it is mocked call + return True + return op(LooseVersion(pkg.__version__), LooseVersion(version)) _IS_WINDOWS = platform.system() == "Windows" From 1d53c77178cb5ac567fe551b5864469424309fb6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 22:33:37 +0100 Subject: [PATCH 09/20] --. --- docs/source/conf.py | 1 + requirements/extra.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 11a0d2a0538bb..6163de976da40 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -334,6 +334,7 @@ def package_list_from_file(file): } MOCK_PACKAGES = [] if SPHINX_MOCK_REQUIREMENTS: + MOCK_PACKAGES += ['fairscale'] # mock also base packages when we are on RTD since we don't install them there MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements.txt')) MOCK_PACKAGES += package_list_from_file(os.path.join(PATH_ROOT, 'requirements', 'extra.txt')) diff --git a/requirements/extra.txt b/requirements/extra.txt index a05c4971ac450..8ec534a5d5e7a 100644 --- a/requirements/extra.txt +++ b/requirements/extra.txt @@ -7,4 +7,5 @@ torchtext>=0.5 # onnx>=1.7.0 onnxruntime>=1.3.0 hydra-core>=1.0 +# todo: when user standard package stream, drop `fairscale` from hard mocked docs libs https://github.com/PyTorchLightning/fairscale/archive/pl_1.2.0.zip From 3b27439fd0da59183ccaef0ee7863be5b5e53acd Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 22:35:47 +0100 Subject: [PATCH 10/20] Apply suggestions from code review --- requirements/extra.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/extra.txt b/requirements/extra.txt index 8ec534a5d5e7a..715916c4e36ac 100644 --- a/requirements/extra.txt +++ b/requirements/extra.txt @@ -7,5 +7,5 @@ torchtext>=0.5 # onnx>=1.7.0 onnxruntime>=1.3.0 hydra-core>=1.0 -# todo: when user standard package stream, drop `fairscale` from hard mocked docs libs +# todo: when switch to standard package stream, drop `fairscale` from hard mocked docs libs https://github.com/PyTorchLightning/fairscale/archive/pl_1.2.0.zip From ee001aa5d2578890f916a03ac98de2fb10d5f008 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 23:01:09 +0100 Subject: [PATCH 11/20] --...- --- pytorch_lightning/utilities/imports.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 53435acef3cec..9f24ffba73237 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -14,6 +14,7 @@ """General utilities""" import importlib import operator +import os import platform import sys from distutils.version import LooseVersion @@ -23,6 +24,9 @@ from pkg_resources import DistributionNotFound +SPHINX_MOCK_REQUIREMENTS = int(os.environ.get('SPHINX_MOCK_REQUIREMENTS', 0)) + + def _module_available(module_path: str) -> bool: """ Check if a path is available in your environment @@ -32,6 +36,8 @@ def _module_available(module_path: str) -> bool: >>> _module_available('bla.bla') False """ + if SPHINX_MOCK_REQUIREMENTS: + return True try: return find_spec(module_path) is not None except AttributeError: @@ -49,6 +55,8 @@ def _compare_version(package: str, op, version) -> bool: >>> _compare_version("torch", operator.ge, "0.1") True """ + if SPHINX_MOCK_REQUIREMENTS: + return True if not _module_available(package): return False try: From 6d37cbd8dce2033c86876c256e18271266bec856 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 23:07:31 +0100 Subject: [PATCH 12/20] --...-- --- pytorch_lightning/utilities/imports.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 9f24ffba73237..86a0f5dabd530 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -36,8 +36,6 @@ def _module_available(module_path: str) -> bool: >>> _module_available('bla.bla') False """ - if SPHINX_MOCK_REQUIREMENTS: - return True try: return find_spec(module_path) is not None except AttributeError: From f8af6af3dc0860a28e532de81c0643c246a46a8f Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 23:41:40 +0100 Subject: [PATCH 13/20] . --- pytorch_lightning/utilities/imports.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 86a0f5dabd530..8b368f91e4e58 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -61,10 +61,12 @@ def _compare_version(package: str, op, version) -> bool: pkg = importlib.import_module(package) except DistributionNotFound: return False - if not hasattr(pkg, '__version__'): + try: + pkg_version = LooseVersion(pkg.__version__) + except (AttributeError, ): # in case version is not defined always return True as likely it is mocked call return True - return op(LooseVersion(pkg.__version__), LooseVersion(version)) + return op(pkg_version, LooseVersion(version)) _IS_WINDOWS = platform.system() == "Windows" From fb79184924245c4013ffc4067d74cae844702f0e Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 23:51:55 +0100 Subject: [PATCH 14/20] .. --- pytorch_lightning/utilities/imports.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 8b368f91e4e58..42badd77f05a6 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -14,7 +14,6 @@ """General utilities""" import importlib import operator -import os import platform import sys from distutils.version import LooseVersion @@ -24,9 +23,6 @@ from pkg_resources import DistributionNotFound -SPHINX_MOCK_REQUIREMENTS = int(os.environ.get('SPHINX_MOCK_REQUIREMENTS', 0)) - - def _module_available(module_path: str) -> bool: """ Check if a path is available in your environment @@ -53,19 +49,18 @@ def _compare_version(package: str, op, version) -> bool: >>> _compare_version("torch", operator.ge, "0.1") True """ - if SPHINX_MOCK_REQUIREMENTS: - return True - if not _module_available(package): - return False try: pkg = importlib.import_module(package) except DistributionNotFound: return False try: pkg_version = LooseVersion(pkg.__version__) - except (AttributeError, ): + except AttributeError: # in case version is not defined always return True as likely it is mocked call return True + if pkg_version.endswith("__version__"): + # this is mock by sphinx, so it shall return True ro generate all summaries + return True return op(pkg_version, LooseVersion(version)) From 9bbab8164784396b6ab8b274815c305326bf76b6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 22 Mar 2021 23:56:11 +0100 Subject: [PATCH 15/20] ..+ --- pytorch_lightning/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 42badd77f05a6..cc1f2527e81c3 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -58,7 +58,7 @@ def _compare_version(package: str, op, version) -> bool: except AttributeError: # in case version is not defined always return True as likely it is mocked call return True - if pkg_version.endswith("__version__"): + if str(pkg_version).endswith("__version__"): # this is mock by sphinx, so it shall return True ro generate all summaries return True return op(pkg_version, LooseVersion(version)) From bf580c03a3cdb2d7c4226a6ed7463b2c9e4c2b2d Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 00:02:56 +0100 Subject: [PATCH 16/20] ..+- --- pytorch_lightning/utilities/imports.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index cc1f2527e81c3..02fff42a53efe 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -56,8 +56,7 @@ def _compare_version(package: str, op, version) -> bool: try: pkg_version = LooseVersion(pkg.__version__) except AttributeError: - # in case version is not defined always return True as likely it is mocked call - return True + return False if str(pkg_version).endswith("__version__"): # this is mock by sphinx, so it shall return True ro generate all summaries return True From b1c484a400017a3e104421b609c2bd3d81be6fa2 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 00:21:58 +0100 Subject: [PATCH 17/20] ..+- --- pytorch_lightning/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 02fff42a53efe..79f0274040e2c 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -51,7 +51,7 @@ def _compare_version(package: str, op, version) -> bool: """ try: pkg = importlib.import_module(package) - except DistributionNotFound: + except (ModuleNotFoundError, DistributionNotFound): return False try: pkg_version = LooseVersion(pkg.__version__) From ef23b0f18717c567a38907b3be4bef377d3a114c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 00:35:13 +0100 Subject: [PATCH 18/20] --keep-going --- .github/workflows/docs-checks.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs-checks.yml b/.github/workflows/docs-checks.yml index 5ee4f23b4b3cc..4488c598c8ac7 100644 --- a/.github/workflows/docs-checks.yml +++ b/.github/workflows/docs-checks.yml @@ -98,7 +98,7 @@ jobs: # First run the same pipeline as Read-The-Docs cd docs make clean - make html --debug --jobs $(nproc) SPHINXOPTS="-W" + make html --debug --jobs $(nproc) SPHINXOPTS="-W --keep-going" - name: Upload built docs uses: actions/upload-artifact@v2 diff --git a/Makefile b/Makefile index d35e0b77f8429..04b08fa2d27d1 100644 --- a/Makefile +++ b/Makefile @@ -29,4 +29,4 @@ test: clean docs: clean pip install --quiet -r requirements/docs.txt - python -m sphinx -b html -W docs/source docs/build + python -m sphinx -b html -W --keep-going docs/source docs/build From 831ee0526ee8b0812d44f30c5610c9f2a2a01ead Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 00:40:14 +0100 Subject: [PATCH 19/20] . --- pytorch_lightning/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index 79f0274040e2c..e76f4e5c67cd0 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -57,7 +57,7 @@ def _compare_version(package: str, op, version) -> bool: pkg_version = LooseVersion(pkg.__version__) except AttributeError: return False - if str(pkg_version).endswith("__version__"): + if pkg.__version__.endswith("__version__"): # this is mock by sphinx, so it shall return True ro generate all summaries return True return op(pkg_version, LooseVersion(version)) From 7696a4077d15399973011fa2d779a86ae13ee2d1 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 23 Mar 2021 08:19:01 +0100 Subject: [PATCH 20/20] .. --- pytorch_lightning/utilities/imports.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/utilities/imports.py b/pytorch_lightning/utilities/imports.py index e76f4e5c67cd0..8090c4ed6590f 100644 --- a/pytorch_lightning/utilities/imports.py +++ b/pytorch_lightning/utilities/imports.py @@ -57,7 +57,7 @@ def _compare_version(package: str, op, version) -> bool: pkg_version = LooseVersion(pkg.__version__) except AttributeError: return False - if pkg.__version__.endswith("__version__"): + if not (hasattr(pkg_version, "vstring") and hasattr(pkg_version, "version")): # this is mock by sphinx, so it shall return True ro generate all summaries return True return op(pkg_version, LooseVersion(version))