support python 3.9 (#4944)

* support python 3.9 * update CI * onnxruntime * . * . * onnxruntime * t 55 * t 75 * add script * use * onnx * onnx * onnx * whl * np * find * 21 * Apply suggestions from code review * Apply suggestions from code review * onnx * CI * req * ~ dockers * min * . * drop horovod * drop horovod * drop horovod * fix * fix * .
Lightning-AI · Mar 29, 2021 · 5b5a5cc · 5b5a5cc
1 parent cca0eca
commit 5b5a5cc
Show file tree

Hide file tree

Showing 12 changed files with 88 additions and 19 deletions.
diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml
@@ -75,8 +75,8 @@ jobs:
             pytorch_version: 1.6
           - python_version: 3.8
             pytorch_version: 1.7
-          #- python_version: 3.9
-          #  pytorch_version: 1.8
+          #  - python_version: 3.9
+          #    pytorch_version: 1.7
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -109,8 +109,10 @@ jobs:
             pytorch_version: 1.4
           - python_version: 3.7
             pytorch_version: 1.7
-          - python_version: 3.7
+          - python_version: 3.8
             pytorch_version: 1.8
+          #  - python_version: 3.9
+          #    pytorch_version: 1.8
     steps:
       - name: Checkout
         uses: actions/checkout@v2

diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml
@@ -17,7 +17,7 @@ jobs:
       matrix:
         # PyTorch 1.5 is failing on Win and bolts requires torchvision>=0.5
         os: [ubuntu-20.04, macOS-10.15 , windows-2019]  #
-        python-version: [3.6, 3.8]
+        python-version: [3.6, 3.9]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml
@@ -16,7 +16,7 @@ jobs:
       # max-parallel: 6
       matrix:
         os: [ubuntu-20.04, windows-2019, macOS-10.15]
-        python-version: [3.7]
+        python-version: [3.8]
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     timeout-minutes: 20

diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml
@@ -15,19 +15,24 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-18.04, windows-2019, macOS-10.15]
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
         requires: ['minimal', 'latest']
+        exclude:
+          - python-version: 3.9
+            requires: 'minimal'
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 35  # TODO: the macOS is taking too long, probably caching did not work...
+    # TODO: the macOS is taking too long, probably caching did not work...
+    timeout-minutes: 40
+
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Update Pip
+    - name: Update pip
       run: |
         # todo: unfreeze PIP after resolving minimal dependencies
         pip install --quiet "pip==20.1" --upgrade --user  # needed for get pip cacher folder
@@ -48,6 +53,19 @@ jobs:
         open(fname, 'w').writelines(lines)
       shell: python
 
+    # todo: re-enable when allow testing py 3.9 with min config, atm some Hydra issues
+    #- name: Adjust minimal for Python 3.9
+    #  if: matrix.requires == 'minimal' && matrix.python-version == 3.9
+    #  run: |
+    #    import re
+    #    def _req(fname, ptn, ver):
+    #        req = re.sub(ptn, ver, open(fname).read())
+    #        open(fname, 'w').write(req)
+    #
+    #    _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0')
+    #    _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0')
+    #  shell: python
+
     - name: Set min. dependencies
       if: matrix.requires == 'minimal'
       run: |
@@ -84,7 +102,6 @@ jobs:
           ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}-
 
     - name: Pull checkpoints from S3
-      # todo: consider adding some caching, but ATM all models have less then 100KB
       run: |
         cd legacy
         # wget is simpler but does not work on Windows
@@ -93,6 +110,13 @@ jobs:
         unzip -o checkpoints.zip
         ls -l checkpoints/
 
+    # todo: re-enable testing with Horovod
+    - name: py3.9 - temp skip Horovod
+      if: matrix.python-version == 3.9
+      run: |
+        # pip uninstall -y horovod
+        python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)"
+
     - name: Install dependencies
       env:
         # MAKEFLAGS: "-j2"
@@ -112,7 +136,8 @@ jobs:
       shell: bash
 
     - name: Reinstall Horovod if necessary
-      if: runner.os != 'windows'
+      # todo: re-enable horovod on py3.9 when it will be supported
+      if: runner.os != 'windows' && matrix.python-version != 3.9
       env:
         HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
       run: |

diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml
@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         python_version: [3.6, 3.7, 3.8]
-        pytorch_version: [1.4, 1.5, 1.6, 1.7]
+        pytorch_version: [1.4, 1.5, 1.6, 1.7, 1.8]
     steps:
       - name: Checkout
         uses: actions/checkout@v2

diff --git a/README.md b/README.md
@@ -79,9 +79,9 @@ Lightning is rigurously tested across multiple GPUs, TPUs CPUs and against major
   | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) |
   | Linux py3.7 [GPUs**] | - | - | [![Build Status](https://dev.azure.com/PytorchLightning/pytorch-lightning/_apis/build/status/PyTorchLightning.pytorch-lightning?branchName=master)](https://dev.azure.com/PytorchLightning/pytorch-lightning/_build/latest?definitionId=2&branchName=master) | - | - |
   | Linux py3.{6,7} [TPUs***] | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) |
-  | Linux py3.{6,7} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
-  | OSX py3.{6,7,8} | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
-  | Windows py3.{6,7,8} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
+  | Linux py3.{6,7,8,9} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
+  | OSX py3.{6,7,8,9} | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
+  | Windows py3.{6,7,8,9} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - |
 
   - _\** tests run on two NVIDIA K80_
   - _\*** tests run on Google GKE TPUv2/3_

diff --git a/requirements/install_AMP.sh → requirements/install_Apex.sh b/requirements/install_AMP.sh → requirements/install_Apex.sh
diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+ROOT=$PWD
+
+#  python -m pip install protobuf
+#  git clone --recursive https://github.com/onnx/onnx.git
+#  cd onnx
+#  python setup.py bdist_wheel
+#  pip install --upgrade dist/*.whl
+#  cd $ROOT
+#  rm -rf onnx
+
+
+# https://github.com/microsoft/onnxruntime/blob/master/BUILD.md
+git clone --recursive https://github.com/Microsoft/onnxruntime
+cd onnxruntime
+export ONNX_ML=1
+pip install setuptools wheel numpy
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+    ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+    # Mac OSX
+    ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel --use_xcode
+elif [[ "$OSTYPE" == "cygwin" ]]; then
+    # POSIX compatibility layer and Linux environment emulation for Windows
+    ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel
+elif [[ "$OSTYPE" == "msys" ]]; then
+    # Lightweight shell and GNU utilities compiled for Windows (part of MinGW)
+    .\build.bat --config RelWithDebInfo --build_shared_lib --build_wheel --parallel
+elif [[ "$OSTYPE" == "win32" ]]; then
+    .\build.bat --config RelWithDebInfo --build_shared_lib --build_wheel --parallel
+else
+    echo $OSTYPE  # Unknown.
+fi
+
+find . -name "*.whl"
+pip install --upgrade $(find . -name "*.whl")
+
+cd $ROOT
+rm -rf onnxruntime
diff --git a/requirements/test.txt b/requirements/test.txt
@@ -6,12 +6,12 @@ pytest>=6.0
 flake8>=3.6
 check-manifest
 twine==3.2
-scikit-learn>=0.22.2
-scikit-image>=0.17.2
 isort>=5.6.4
 mypy>=0.720, <0.800
 pre-commit>=1.0
 
 cloudpickle>=1.3
+scikit-learn>0.22.1
+scikit-image>0.17.1
 nltk>=3.3
 pandas  # needed in benchmarks
diff --git a/setup.py b/setup.py
@@ -110,5 +110,6 @@
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
     ],
 )
diff --git a/tests/README.md b/tests/README.md
@@ -17,7 +17,7 @@ git clone https://github.com/PyTorchLightning/pytorch-lightning
 cd pytorch-lightning
 
 # install AMP support
-bash requirements/install_AMP.sh
+bash requirements/install_Apex.sh
 
 # install dev deps
 pip install -r requirements/devel.txt

diff --git a/tests/models/test_horovod.py b/tests/models/test_horovod.py
@@ -63,7 +63,7 @@ def _run_horovod(trainer_options, on_gpu=False):
     assert exit_code == 0
 
 
-@RunIf(skip_windows=True)
+@RunIf(skip_windows=True, horovod=True)
 def test_horovod_cpu(tmpdir):
     """Test Horovod running multi-process on CPU."""
     trainer_options = dict(
@@ -80,7 +80,7 @@ def test_horovod_cpu(tmpdir):
     _run_horovod(trainer_options)
 
 
-@RunIf(skip_windows=True)
+@RunIf(skip_windows=True, horovod=True)
 def test_horovod_cpu_implicit(tmpdir):
     """Test Horovod without specifying a backend, inferring from env set by `horovodrun`."""
     trainer_options = dict(