Merge dask/main into martindurant/cy_scheduler_state

.coveragerc

@@ -1,19 +1,32 @@
 [run]
-include = 
-    distributed/*
+source =
+    distributed
 omit =
     distributed/tests/test*
-    distributed/hdfs.py
-    distributed/cluster.py
     distributed/*/tests/test*
-    distributed/compatibility.py
     distributed/cli/utils.py
-    distributed/utils_test.py
+    distributed/cli/dask_spec.py
     distributed/deploy/ssh.py
     distributed/_ipython_utils.py
+    distributed/_version.py
+    distributed/pytest_resourceleaks.py
+    distributed/comm/ucx.py
+    distributed/asyncio.py
 
 [report]
 show_missing = True
+exclude_lines =
+    # re-enable the standard pragma
+    pragma: nocover
+    pragma: no cover
+    # exclude nvml calls
+    [\s(.]nvml[\s(.]
+    [\s(.]pynvml[\s(.]
+    # exclude LOG_PDB
+    LOG_PDB
+    # always ignore type checking blocks
+    TYPE_CHECKING
+    except ImportError
 
 [html]
 directory = coverage_html_report

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,3 @@
 - [ ] Closes #xxxx
 - [ ] Tests added / passed
-- [ ] Passes `black distributed` / `flake8 distributed` / `isort distributed`
+- [ ] Passes `pre-commit run --all-files`

.github/workflows/conda.yml

@@ -0,0 +1,86 @@
+name: Conda build
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs from the same branch
+concurrency:
+  group: conda-${{ github.head_ref }}
+  cancel-in-progress: true
+
+# Required shell entrypoint to have properly activated conda environments
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  conda:
+    name: Build (and upload)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          use-mamba: true
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          mamba install boa conda-verify
+
+          which python
+          pip list
+          mamba list
+      - name: Build conda packages
+        run: |
+          # suffix for pre-release package versions
+          export VERSION_SUFFIX=a`date +%y%m%d`
+
+          # conda search for the latest dask-core pre-release
+          arr=($(conda search --override-channels -c dask/label/dev dask-core | tail -n 1))
+
+          # extract dask-core pre-release version / build
+          export DASK_CORE_VERSION=${arr[1]}
+
+          # distributed pre-release build
+          conda mambabuild continuous_integration/recipes/distributed \
+                           --channel dask/label/dev \
+                           --no-anaconda-upload \
+                           --output-folder build
+
+          # dask pre-release build
+          conda mambabuild continuous_integration/recipes/dask \
+                           --channel dask/label/dev \
+                           --no-anaconda-upload \
+                           --output-folder build
+      - name: Upload conda packages
+        if: |
+          github.event_name == 'push'
+          && github.ref == 'refs/heads/main'
+          && github.repository == 'dask/distributed'
+        env:
+          ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }}
+        run: |
+          # convert distributed to other architectures
+          cd build && conda convert linux-64/*.tar.bz2 -p osx-64 \
+                                                       -p osx-arm64 \
+                                                       -p linux-ppc64le \
+                                                       -p linux-aarch64 \
+                                                       -p win-64
+
+          # install anaconda for upload
+          mamba install anaconda-client
+
+          anaconda upload --label dev noarch/*.tar.bz2
+          anaconda upload --label dev linux-64/*.tar.bz2
+          anaconda upload --label dev linux-aarch64/*.tar.bz2
+          anaconda upload --label dev linux-ppc64le/*.tar.bz2
+          anaconda upload --label dev osx-64/*.tar.bz2
+          anaconda upload --label dev osx-arm64/*.tar.bz2
+          anaconda upload --label dev win-64/*.tar.bz2

.github/workflows/publish-test-results.yaml

@@ -0,0 +1,45 @@
+# Copied from https://github.com/EnricoMi/publish-unit-test-result-action/blob/v1.23/README.md#support-fork-repositories-and-dependabot-branches
+# Warning: changes to this workflow will NOT be picked up until they land in the main branch!
+# See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
+
+name: Publish test results
+
+on:
+  workflow_run:
+    workflows: [Tests]
+    types: [completed]
+
+jobs:
+  publish-test-results:
+    name: Publish test results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+
+    # Needed to post comments on the PR
+    permissions:
+      checks: write
+      pull-requests: write
+
+    steps:
+      - name: Download and extract artifacts
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: |
+          mkdir artifacts && cd artifacts
+
+          artifacts_url=${{ github.event.workflow_run.artifacts_url }}
+
+          gh api "$artifacts_url" -q '.artifacts[] | [.name, .archive_download_url] | @tsv' | while read artifact
+          do
+            IFS=$'\t' read name url <<< "$artifact"
+            gh api $url > "$name.zip"
+            unzip -d "$name" "$name.zip"
+          done
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        with:
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: artifacts/**/*.xml

.github/workflows/ssh_debug.yaml → .github/workflows/ssh_debug.yaml.donotrun

@@ -1,7 +1,6 @@
 name: Debug passwordless `ssh localhost`
 
-on: []
-# on: [pull_request]  # Uncomment to enable
+on: [push, pull_request]
 
 jobs:
   test:

.github/workflows/test-report.yaml

@@ -0,0 +1,47 @@
+name: Test Report
+
+on:
+  schedule:
+    - cron: "47 6 * * *"
+  workflow_dispatch:
+
+jobs:
+  test-report:
+    name: Test Report
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ github.token }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          condarc-file: continuous_integration/condarc
+          use-mamba: true
+          python-version: 3.9
+          environment-file: continuous_integration/scripts/test-report-environment.yml
+          activate-environment: dask-distributed
+
+      - name: Show conda options
+        shell: bash -l {0}
+        run: conda config --show
+
+      - name: mamba list
+        shell: bash -l {0}
+        run: mamba list
+
+      - name: Generate report
+        shell: bash -l {0}
+        run: |
+          python continuous_integration/scripts/test_report.py
+          mkdir deploy
+          mv test_report.html deploy/
+
+      - name: Deploy 🚀
+        uses: JamesIves/[email protected]
+        with:
+          branch: gh-pages
+          folder: deploy

.github/workflows/tests.yaml

@@ -1,6 +1,10 @@
 name: Tests
 
-on: [push, pull_request]
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: "0 6,18 * * *"
 
 # When this workflow is queued, automatically cancel any previous running
 # or pending jobs from the same branch
@@ -10,16 +14,23 @@ concurrency:
 
 jobs:
   test:
+    # Do not run the schedule job on forks
+    if: github.repository == 'dask/distributed' || github.event_name != 'schedule'
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
 
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.8", "3.9"]
         # Cherry-pick test modules to split the overall runtime roughly in half
         partition: [ci1, not ci1]
+        include:
+          - partition: "ci1"
+            partition-label: "ci1"
+          - partition: "not ci1"
+            partition-label: "notci1"
 
         # Uncomment to stress-test the test suite for random failures.
         # Must also change env.TEST_ID below.
@@ -30,8 +41,8 @@ jobs:
         # run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
 
     env:
-      TEST_ID: ${{ matrix.os }}-${{ matrix.python-version }}
-      # TEST_ID: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.run }}
+      TEST_ID: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.partition-label }}
+      # TEST_ID: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.partition-label }}-${{ matrix.run }}
 
     steps:
       - name: Checkout source
@@ -70,8 +81,8 @@ jobs:
       - name: Install
         shell: bash -l {0}
         run: |
-          # Cythonize scheduler on Python 3.7 builds
-          if [[ "${{ matrix.python-version }}" = "3.7" ]]; then
+          # Cythonize scheduler on Python 3.8 builds
+          if [[ "${{ matrix.python-version }}" = "3.8" ]]; then
               python -m pip install -vv --no-deps --install-option="--with-cython" -e .
               python -c "from distributed.scheduler import COMPILED; assert COMPILED"
           else
@@ -115,14 +126,20 @@ jobs:
           fi
           source continuous_integration/scripts/set_ulimit.sh
 
-          pytest distributed -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
-            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID
+          pytest distributed \
+            -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
+            --leaks=fds,processes,threads \
+            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID \
+            --cov=distributed --cov-report=xml
 
       # - name: Debug with tmate on failure
       #   if: ${{ failure() }}
       #   uses: mxschmitt/action-tmate@v3
 
-      - name: Upload test artifacts
+      - name: Coverage
+        uses: codecov/codecov-action@v1
+
+      - name: Upload test results
         # ensure this runs even if pytest fails
         if: >
           always() &&
@@ -131,3 +148,26 @@ jobs:
         with:
           name: ${{ env.TEST_ID }}
           path: reports
+      - name: Upload gen_cluster dumps for failed tests
+        # ensure this runs even if pytest fails
+        if: >
+          always() &&
+          (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.TEST_ID }}_cluster_dumps
+          path: test_cluster_dump
+          if-no-files-found: ignore
+
+  # Publish an artifact for the event; used by publish-test-results.yaml
+  event_file:
+    # Do not run the schedule job on forks
+    if: github.repository == 'dask/distributed' || github.event_name != 'schedule'
+    name: "Event File"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v2
+        with:
+          name: Event File
+          path: ${{ github.event_path }}

.github/workflows/update-gpuci.yaml

@@ -0,0 +1,51 @@
+name: Check for gpuCI updates
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # Daily “At 00:00” UTC
+  workflow_dispatch:
+
+jobs:
+  update-gpuci:
+    runs-on: ubuntu-latest
+    if: github.repository == 'dask/distributed'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Get latest cuDF nightly version
+        id: latest_version
+        uses: jacobtomlinson/[email protected]
+        with:
+          org: "rapidsai-nightly"
+          package: "cudf"
+          version_system: "CalVer"
+
+      - name: Strip git tags from versions
+        env:
+          FULL_RAPIDS_VER: ${{ steps.latest_version.outputs.version }}
+        run: echo "RAPIDS_VER=${FULL_RAPIDS_VER::-10}" >> $GITHUB_ENV
+
+      - name: Find and Replace Release
+        uses: jacobtomlinson/[email protected]
+        with:
+          include: 'continuous_integration\/gpuci\/axis\.yaml'
+          find: "RAPIDS_VER:\n- .*"
+          replace: |-
+            RAPIDS_VER:
+            - "${{ env.RAPIDS_VER }}"
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: true
+          commit-message: "Update gpuCI `RAPIDS_VER` to `${{ env.RAPIDS_VER }}`"
+          title: "Update gpuCI `RAPIDS_VER` to `${{ env.RAPIDS_VER }}`"
+          team-reviewers: "dask/gpu"
+          author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
+          branch: "upgrade-gpuci-rapids"
+          body: |
+            A new cuDF nightly version has been detected.
+
+            Updated `axis.yaml` to use `${{ env.RAPIDS_VER }}`.