From 5b5a5cc80b7a752f952c7450b005d345b824fd0b Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 29 Mar 2021 18:20:13 +0200 Subject: [PATCH] support python 3.9 (#4944) * support python 3.9 * update CI * onnxruntime * . * . * onnxruntime * t 55 * t 75 * add script * use * onnx * onnx * onnx * whl * np * find * 21 * Apply suggestions from code review * Apply suggestions from code review * onnx * CI * req * ~ dockers * min * . * drop horovod * drop horovod * drop horovod * fix * fix * . --- .github/workflows/ci_dockers.yml | 8 ++-- .github/workflows/ci_pkg-install.yml | 2 +- .github/workflows/ci_test-base.yml | 2 +- .github/workflows/ci_test-full.yml | 35 +++++++++++++--- .github/workflows/release-docker.yml | 2 +- README.md | 6 +-- .../{install_AMP.sh => install_Apex.sh} | 0 requirements/install_ONNX.sh | 41 +++++++++++++++++++ requirements/test.txt | 4 +- setup.py | 1 + tests/README.md | 2 +- tests/models/test_horovod.py | 4 +- 12 files changed, 88 insertions(+), 19 deletions(-) rename requirements/{install_AMP.sh => install_Apex.sh} (100%) create mode 100644 requirements/install_ONNX.sh diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index 897e16a12d44f..a22be81b398fb 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -75,8 +75,8 @@ jobs: pytorch_version: 1.6 - python_version: 3.8 pytorch_version: 1.7 - #- python_version: 3.9 - # pytorch_version: 1.8 + # - python_version: 3.9 + # pytorch_version: 1.7 steps: - name: Checkout uses: actions/checkout@v2 @@ -109,8 +109,10 @@ jobs: pytorch_version: 1.4 - python_version: 3.7 pytorch_version: 1.7 - - python_version: 3.7 + - python_version: 3.8 pytorch_version: 1.8 + # - python_version: 3.9 + # pytorch_version: 1.8 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml index 54c9f5c007c82..b4557e5ed75aa 100644 --- a/.github/workflows/ci_pkg-install.yml +++ b/.github/workflows/ci_pkg-install.yml @@ -17,7 +17,7 @@ jobs: matrix: # PyTorch 1.5 is failing on Win and bolts requires torchvision>=0.5 os: [ubuntu-20.04, macOS-10.15 , windows-2019] # - python-version: [3.6, 3.8] + python-version: [3.6, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml index 77363992718af..cf92c64f1feec 100644 --- a/.github/workflows/ci_test-base.yml +++ b/.github/workflows/ci_test-base.yml @@ -16,7 +16,7 @@ jobs: # max-parallel: 6 matrix: os: [ubuntu-20.04, windows-2019, macOS-10.15] - python-version: [3.7] + python-version: [3.8] # Timeout: https://stackoverflow.com/a/59076067/4521646 timeout-minutes: 20 diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 5a3e23a37fd0b..ec9e71c5b83b2 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -15,11 +15,16 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, windows-2019, macOS-10.15] - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] requires: ['minimal', 'latest'] + exclude: + - python-version: 3.9 + requires: 'minimal' # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 35 # TODO: the macOS is taking too long, probably caching did not work... + # TODO: the macOS is taking too long, probably caching did not work... + timeout-minutes: 40 + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -27,7 +32,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Update Pip + - name: Update pip run: | # todo: unfreeze PIP after resolving minimal dependencies pip install --quiet "pip==20.1" --upgrade --user # needed for get pip cacher folder @@ -48,6 +53,19 @@ jobs: open(fname, 'w').writelines(lines) shell: python + # todo: re-enable when allow testing py 3.9 with min config, atm some Hydra issues + #- name: Adjust minimal for Python 3.9 + # if: matrix.requires == 'minimal' && matrix.python-version == 3.9 + # run: | + # import re + # def _req(fname, ptn, ver): + # req = re.sub(ptn, ver, open(fname).read()) + # open(fname, 'w').write(req) + # + # _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0') + # _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0') + # shell: python + - name: Set min. dependencies if: matrix.requires == 'minimal' run: | @@ -84,7 +102,6 @@ jobs: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- - name: Pull checkpoints from S3 - # todo: consider adding some caching, but ATM all models have less then 100KB run: | cd legacy # wget is simpler but does not work on Windows @@ -93,6 +110,13 @@ jobs: unzip -o checkpoints.zip ls -l checkpoints/ + # todo: re-enable testing with Horovod + - name: py3.9 - temp skip Horovod + if: matrix.python-version == 3.9 + run: | + # pip uninstall -y horovod + python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" + - name: Install dependencies env: # MAKEFLAGS: "-j2" @@ -112,7 +136,8 @@ jobs: shell: bash - name: Reinstall Horovod if necessary - if: runner.os != 'windows' + # todo: re-enable horovod on py3.9 when it will be supported + if: runner.os != 'windows' && matrix.python-version != 3.9 env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma" run: | diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 36ecbe229ac7c..4838ac915bd47 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -14,7 +14,7 @@ jobs: fail-fast: false matrix: python_version: [3.6, 3.7, 3.8] - pytorch_version: [1.4, 1.5, 1.6, 1.7] + pytorch_version: [1.4, 1.5, 1.6, 1.7, 1.8] steps: - name: Checkout uses: actions/checkout@v2 diff --git a/README.md b/README.md index dd30bd94d78a6..cbe1341df2f55 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,9 @@ Lightning is rigurously tested across multiple GPUs, TPUs CPUs and against major | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | | Linux py3.7 [GPUs**] | - | - | [![Build Status](https://dev.azure.com/PytorchLightning/pytorch-lightning/_apis/build/status/PyTorchLightning.pytorch-lightning?branchName=master)](https://dev.azure.com/PytorchLightning/pytorch-lightning/_build/latest?definitionId=2&branchName=master) | - | - | | Linux py3.{6,7} [TPUs***] | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - | Linux py3.{6,7} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | OSX py3.{6,7,8} | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | Windows py3.{6,7,8} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | + | Linux py3.{6,7,8,9} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | + | OSX py3.{6,7,8,9} | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | + | Windows py3.{6,7,8,9} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - _\** tests run on two NVIDIA K80_ - _\*** tests run on Google GKE TPUv2/3_ diff --git a/requirements/install_AMP.sh b/requirements/install_Apex.sh similarity index 100% rename from requirements/install_AMP.sh rename to requirements/install_Apex.sh diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh new file mode 100644 index 0000000000000..d6784fa373d6f --- /dev/null +++ b/requirements/install_ONNX.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +ROOT=$PWD + +# python -m pip install protobuf +# git clone --recursive https://github.com/onnx/onnx.git +# cd onnx +# python setup.py bdist_wheel +# pip install --upgrade dist/*.whl +# cd $ROOT +# rm -rf onnx + + +# https://github.com/microsoft/onnxruntime/blob/master/BUILD.md +git clone --recursive https://github.com/Microsoft/onnxruntime +cd onnxruntime +export ONNX_ML=1 +pip install setuptools wheel numpy + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel +elif [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel --use_xcode +elif [[ "$OSTYPE" == "cygwin" ]]; then + # POSIX compatibility layer and Linux environment emulation for Windows + ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel +elif [[ "$OSTYPE" == "msys" ]]; then + # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) + .\build.bat --config RelWithDebInfo --build_shared_lib --build_wheel --parallel +elif [[ "$OSTYPE" == "win32" ]]; then + .\build.bat --config RelWithDebInfo --build_shared_lib --build_wheel --parallel +else + echo $OSTYPE # Unknown. +fi + +find . -name "*.whl" +pip install --upgrade $(find . -name "*.whl") + +cd $ROOT +rm -rf onnxruntime diff --git a/requirements/test.txt b/requirements/test.txt index 259cc2e2d6442..3c81479e148c2 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,12 +6,12 @@ pytest>=6.0 flake8>=3.6 check-manifest twine==3.2 -scikit-learn>=0.22.2 -scikit-image>=0.17.2 isort>=5.6.4 mypy>=0.720, <0.800 pre-commit>=1.0 cloudpickle>=1.3 +scikit-learn>0.22.1 +scikit-image>0.17.1 nltk>=3.3 pandas # needed in benchmarks diff --git a/setup.py b/setup.py index e53e24ebf0702..0979de8ce9065 100755 --- a/setup.py +++ b/setup.py @@ -110,5 +110,6 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], ) diff --git a/tests/README.md b/tests/README.md index 7b857a1901fd7..0b0563a3ae540 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,7 +17,7 @@ git clone https://github.com/PyTorchLightning/pytorch-lightning cd pytorch-lightning # install AMP support -bash requirements/install_AMP.sh +bash requirements/install_Apex.sh # install dev deps pip install -r requirements/devel.txt diff --git a/tests/models/test_horovod.py b/tests/models/test_horovod.py index 3c8c9b0f36041..5b4a700babd1d 100644 --- a/tests/models/test_horovod.py +++ b/tests/models/test_horovod.py @@ -63,7 +63,7 @@ def _run_horovod(trainer_options, on_gpu=False): assert exit_code == 0 -@RunIf(skip_windows=True) +@RunIf(skip_windows=True, horovod=True) def test_horovod_cpu(tmpdir): """Test Horovod running multi-process on CPU.""" trainer_options = dict( @@ -80,7 +80,7 @@ def test_horovod_cpu(tmpdir): _run_horovod(trainer_options) -@RunIf(skip_windows=True) +@RunIf(skip_windows=True, horovod=True) def test_horovod_cpu_implicit(tmpdir): """Test Horovod without specifying a backend, inferring from env set by `horovodrun`.""" trainer_options = dict(