Drop Python 3.7 (#5683)

.github/workflows/tests.yaml

@@ -23,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.8", "3.9"]
         # Cherry-pick test modules to split the overall runtime roughly in half
         partition: [ci1, not ci1]
         include:
@@ -81,8 +81,8 @@ jobs:
       - name: Install
         shell: bash -l {0}
         run: |
-          # Cythonize scheduler on Python 3.7 builds
-          if [[ "${{ matrix.python-version }}" = "3.7" ]]; then
+          # Cythonize scheduler on Python 3.8 builds
+          if [[ "${{ matrix.python-version }}" = "3.8" ]]; then
               python -m pip install -vv --no-deps --install-option="--with-cython" -e .
               python -c "from distributed.scheduler import COMPILED; assert COMPILED"
           else

.pre-commit-config.yaml

@@ -11,15 +11,15 @@ repos:
     hooks:
       - id: pyupgrade
         args:
-          - --py37-plus
+          - --py38-plus
   -   repo: https://github.com/psf/black
       rev: 22.1.0
       hooks:
       - id: black
         language_version: python3
         exclude: versioneer.py
         args:
-          - --target-version=py37
+          - --target-version=py38
   -   repo: https://gitlab.com/pycqa/flake8
       rev: 3.9.2
       hooks:
@@ -38,8 +38,6 @@ repos:
           - types-PyYAML
           - types-setuptools
           - types-psutil
-          # Libraries exclusively imported under `if TYPE_CHECKING:`
-          - typing_extensions  # To be reviewed after dropping Python 3.7
           # Typed libraries
           - numpy
           - dask

.readthedocs.yaml

@@ -0,0 +1,15 @@
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+version: 2
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+
+sphinx:
+  configuration: docs/source/conf.py
+
+python:
+  install:
+    - requirements: docs/requirements.txt
+    - method: pip
+      path: .

continuous_integration/environment-3.8.yaml

@@ -11,6 +11,7 @@ dependencies:
   - click
   - cloudpickle
   - coverage
+  - cython  # Only tested here; also a dependency of crick
   - dask  # overridden by git tip below
   - filesystem-spec
   - h5py
@@ -44,4 +45,5 @@ dependencies:
   - zstandard
   - pip:
       - git+https://github.com/dask/dask
+      - git+https://github.com/jcrist/crick  # Only tested here
       - keras

continuous_integration/recipes/dask/meta.yaml

@@ -18,9 +18,9 @@ build:
 
 requirements:
   host:
-    - python >=3.7
+    - python >=3.8
   run:
-    - python >=3.7
+    - python >=3.8
     - dask-core >={{ dask_version }}
     - distributed {{ version }}=*_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
     - cytoolz >=0.8.2

continuous_integration/recipes/distributed/conda_build_config.yaml

@@ -1,4 +1,3 @@
 python:
-    - 3.7
     - 3.8
     - 3.9

distributed/client.py

@@ -24,13 +24,10 @@
 from functools import partial
 from numbers import Number
 from queue import Queue as pyQueue
-from typing import TYPE_CHECKING, ClassVar
+from typing import ClassVar, Literal
 
 from tlz import first, groupby, keymap, merge, partition_all, valmap
 
-if TYPE_CHECKING:
-    from typing_extensions import Literal
-
 import dask
 from dask.base import collections_to_dsk, normalize_token, tokenize
 from dask.core import flatten

distributed/comm/tests/test_comms.py

@@ -769,10 +769,6 @@ async def test_tcp_comm_closed_explicit(tcp):
     await check_comm_closed_explicit("tcp://127.0.0.1")
 
 
-@pytest.mark.xfail(
-    sys.version_info[:2] == (3, 7),
-    reason="This test fails on python 3.7 with certain versions of openssl",
-)
 @pytest.mark.asyncio
 async def test_tls_comm_closed_explicit(tcp):
     await check_comm_closed_explicit("tls://127.0.0.1", **tls_kwargs)

distributed/compatibility.py

@@ -4,8 +4,6 @@
 import platform
 import sys
 
-import tornado
-
 logging_names: dict[str | int, int | str] = {}
 logging_names.update(logging._levelToName)  # type: ignore
 logging_names.update(logging._nameToLevel)  # type: ignore
@@ -14,4 +12,3 @@
 LINUX = sys.platform == "linux"
 MACOS = sys.platform == "darwin"
 WINDOWS = sys.platform.startswith("win")
-TORNADO6 = tornado.version_info[0] >= 6

distributed/deploy/tests/test_local.py

@@ -9,8 +9,6 @@
 from urllib.parse import urlparse
 
 import pytest
-import tornado
-from packaging.version import parse as parse_version
 from tornado.httpclient import AsyncHTTPClient
 from tornado.ioloop import IOLoop
 
@@ -455,13 +453,6 @@ async def test_scale_up_and_down():
             assert len(cluster.workers) == 1
 
 
-@pytest.mark.xfail(
-    sys.version_info >= (3, 8)
-    and parse_version(tornado.version) < parse_version("6.0.3"),
-    reason="Known issue with Python 3.8 and Tornado < 6.0.3. "
-    "See https://github.com/tornadoweb/tornado/pull/2683.",
-    strict=True,
-)
 def test_silent_startup():
     code = """if 1:
         from time import sleep

distributed/protocol/compression.py

@@ -10,7 +10,7 @@
 from collections.abc import Callable
 from contextlib import suppress
 from functools import partial
-from typing import TYPE_CHECKING
+from typing import Literal
 
 from tlz import identity
 
@@ -27,9 +27,6 @@
 
 from ..utils import ensure_bytes
 
-if TYPE_CHECKING:
-    from typing_extensions import Literal
-
 compressions: dict[
     str | None | Literal[False],
     dict[Literal["compress", "decompress"], Callable[[bytes], bytes]],

distributed/protocol/pickle.py

@@ -1,17 +1,8 @@
 import logging
-import sys
+import pickle
 
 import cloudpickle
 
-if sys.version_info < (3, 8):
-    try:
-        import pickle5 as pickle
-    except ImportError:
-        import pickle
-else:
-    import pickle
-
-
 HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
 
 logger = logging.getLogger(__name__)

distributed/protocol/tests/test_pickle.py

@@ -1,5 +1,5 @@
 import gc
-import sys
+import pickle
 import weakref
 from functools import partial
 from operator import add
@@ -9,14 +9,6 @@
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 
-if sys.version_info < (3, 8):
-    try:
-        import pickle5 as pickle
-    except ImportError:
-        import pickle
-else:
-    import pickle
-
 
 class MemoryviewHolder:
     def __init__(self, mv):

distributed/scheduler.py

@@ -7,6 +7,7 @@
 import math
 import operator
 import os
+import pickle
 import random
 import sys
 import uuid
@@ -26,12 +27,9 @@
 from datetime import timedelta
 from functools import partial
 from numbers import Number
-from typing import TYPE_CHECKING, ClassVar, Container
+from typing import ClassVar, Container, Literal
 from typing import cast as pep484_cast
 
-if TYPE_CHECKING:
-    from typing_extensions import Literal
-
 import psutil
 from sortedcontainers import SortedDict, SortedSet
 from tlz import (
@@ -161,15 +159,6 @@ def nogil(func):
         return func
 
 
-if sys.version_info < (3, 8):
-    try:
-        import pickle5 as pickle
-    except ImportError:
-        import pickle
-else:
-    import pickle
-
-
 logger = logging.getLogger(__name__)
 
 

distributed/tests/test_scheduler.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import operator
+import pickle
 import re
 import sys
 from itertools import product
@@ -43,14 +44,6 @@
 )
 from distributed.worker import dumps_function, dumps_task, get_worker
 
-if sys.version_info < (3, 8):
-    try:
-        import pickle5 as pickle
-    except ImportError:
-        import pickle
-else:
-    import pickle
-
 pytestmark = pytest.mark.ci1
 
 

distributed/tests/test_steal.py

@@ -3,7 +3,6 @@
 import itertools
 import logging
 import random
-import sys
 import weakref
 from operator import mul
 from time import sleep
@@ -14,7 +13,7 @@
 import dask
 
 from distributed import Lock, Nanny, Worker, wait, worker_client
-from distributed.compatibility import LINUX, WINDOWS
+from distributed.compatibility import LINUX
 from distributed.config import config
 from distributed.core import Status
 from distributed.metrics import time
@@ -799,12 +798,6 @@ async def test_steal_communication_heavy_tasks(c, s, a, b):
     assert s.processing[b.address]
 
 
-@pytest.mark.flaky(
-    condition=WINDOWS and sys.version_info[:2] == (3, 7),
-    reruns=20,
-    reruns_delay=5,
-    reason="b.in_flight_tasks == 1",
-)
 @gen_cluster(client=True)
 async def test_steal_twice(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)

distributed/utils.py

@@ -1126,10 +1126,6 @@ def color_of(x, palette=palette):
 
 
 def _iscoroutinefunction(f):
-    # Python < 3.8 does not support determining if `partial` objects wrap async funcs
-    if sys.version_info < (3, 8):
-        while isinstance(f, functools.partial):
-            f = f.func
     return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
 
 

distributed/utils_test.py

@@ -27,10 +27,7 @@
 from glob import glob
 from itertools import count
 from time import sleep
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    from typing_extensions import Literal
+from typing import Any, Literal
 
 from distributed.compatibility import MACOS
 from distributed.scheduler import Scheduler

distributed/worker.py

@@ -26,10 +26,9 @@
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import TYPE_CHECKING, Any, ClassVar, Container
+from typing import TYPE_CHECKING, Any, ClassVar, Container, Literal
 
 if TYPE_CHECKING:
-    from typing_extensions import Literal
     from .diagnostics.plugin import WorkerPlugin
     from .actor import Actor
     from .client import Client