diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh index 62eeb4d131b..f307b7e0451 100755 --- a/ci/benchmark/build.sh +++ b/ci/benchmark/build.sh @@ -36,8 +36,8 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" # like `/tmp` is. export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" -# Dask & Distributed git tag -export DASK_DISTRIBUTED_GIT_TAG='2022.01.0' +# Dask & Distributed option to install main(nightly) or `conda-forge` packages. +export INSTALL_DASK_MAIN=1 function remove_libcudf_kernel_cache_dir { EXITCODE=$? @@ -77,11 +77,16 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ # conda remove -f rapids-build-env rapids-notebook-env # conda install "your-pkg=1.0.0" -# Install the master version of dask, distributed, and streamz -logger "pip install git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps" -pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps -logger "pip install git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps" -pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps +# Install the conda-forge or nightly version of dask and distributed +if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then + gpuci_logger "gpuci_mamba_retry update dask" + gpuci_mamba_retry update dask +else + gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 conda-forge::dask-core>=2022.02.1 --force-reinstall" + gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 conda-forge::dask-core>=2022.02.1 --force-reinstall +fi + +# Install the master version of streamz logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps" pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 574a55d26b6..465a6eae7e4 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. ############################################## # cuDF CPU conda build script for CI # ############################################## @@ -46,9 +46,10 @@ gpuci_logger "Activate conda env" . /opt/conda/etc/profile.d/conda.sh conda activate rapids -# Remove rapidsai-nightly channel if we are building main branch +# Remove `rapidsai-nightly` & `dask/label/dev` channel if we are building main branch if [ "$SOURCE_BRANCH" = "main" ]; then conda config --system --remove channels rapidsai-nightly + conda config --system --remove channels dask/label/dev fi gpuci_logger "Check compiler versions" diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index aa57d02032b..4492ee1d443 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -30,8 +30,8 @@ export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` -# Dask & Distributed git tag -export DASK_DISTRIBUTED_GIT_TAG='2022.01.0' +# Dask & Distributed option to install main(nightly) or `conda-forge` packages. +export INSTALL_DASK_MAIN=1 # ucx-py version export UCX_PY_VERSION='0.25.*' @@ -108,11 +108,19 @@ conda config --show-sources conda list --show-channel-urls function install_dask { - # Install the main version of dask, distributed, and streamz - gpuci_logger "Install the main version of dask, distributed, and streamz" + # Install the conda-forge or nightly version of dask and distributed + gpuci_logger "Install the conda-forge or nightly version of dask and distributed" set -x - pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps - pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps + if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then + gpuci_logger "gpuci_mamba_retry update dask" + gpuci_mamba_retry update dask + conda list + else + gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 conda-forge::dask-core>=2022.02.1 --force-reinstall" + gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 conda-forge::dask-core>=2022.02.1 --force-reinstall + fi + # Install the main version of streamz + gpuci_logger "Install the main version of streamz" # Need to uninstall streamz that is already in the env. pip uninstall -y streamz pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index b926a6cdc99..097ca2089a5 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -5,6 +5,7 @@ channels: - rapidsai - nvidia - rapidsai-nightly + - dask/label/dev - conda-forge dependencies: - clang=11.1.0 @@ -42,8 +43,8 @@ dependencies: - pydocstyle=6.1.1 - typing_extensions - pre-commit - - dask>=2021.11.1,<=2022.01.0 - - distributed>=2021.11.1,<=2022.01.0 + - dask>=2022.02.1 + - distributed>=2022.02.1 - streamz - arrow-cpp=6.0.1 - dlpack>=0.5,<0.6.0a0 @@ -63,8 +64,6 @@ dependencies: - librdkafka=1.7.0 - python-confluent-kafka=1.7.0 - pip: - - git+https://github.com/dask/dask.git@main - - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - ptxcompiler # [linux64] diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index 2e8badc3a54..de51e8a205c 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -32,8 +32,8 @@ requirements: - python - streamz - cudf {{ version }} - - dask>=2021.11.1,<=2022.01.0 - - distributed>=2021.11.1,<=2022.01.0 + - dask>=2022.02.1 + - distributed>=2022.02.1 - python-confluent-kafka >=1.7.0,<1.8.0a0 - cudf_kafka {{ version }} diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 225d77729df..fe565cf8848 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2019, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} @@ -27,14 +27,14 @@ requirements: host: - python - cudf {{ version }} - - dask>=2021.11.1,<=2022.01.0 - - distributed>=2021.11.1,<=2022.01.0 + - dask>=2022.02.1 + - distributed>=2022.02.1 - cudatoolkit {{ cuda_version }} run: - python - cudf {{ version }} - - dask>=2021.11.1,<=2022.01.0 - - distributed>=2021.11.1,<=2022.01.0 + - dask>=2022.02.1 + - distributed>=2022.02.1 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} test: # [linux64] diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt index b4cd7a649ee..18a16e15248 100644 --- a/python/custreamz/dev_requirements.txt +++ b/python/custreamz/dev_requirements.txt @@ -3,8 +3,8 @@ flake8==3.8.3 black==19.10b0 isort==5.6.4 -dask>=2021.11.1,<=2022.01.0 -distributed>=2021.11.1,<=2022.01.0 +dask>=2022.02.1 +distributed>=2022.02.1 streamz python-confluent-kafka pytest diff --git a/python/dask_cudf/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt index d5ba79d4987..88d535cb906 100644 --- a/python/dask_cudf/dev_requirements.txt +++ b/python/dask_cudf/dev_requirements.txt @@ -1,7 +1,7 @@ # Copyright (c) 2021, NVIDIA CORPORATION. -dask>=2021.11.1,<=2022.01.0 -distributed>=2021.11.1,<=2022.01.0 +dask>=2022.02.1 +distributed>=2022.02.1 fsspec>=0.6.0 numba>=0.53.1 numpy diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 635f21fd906..412321e2281 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. import os import re @@ -10,8 +10,8 @@ install_requires = [ "cudf", - "dask>=2021.11.1,<=2022.01.0", - "distributed>=2021.11.1,<=2022.01.0", + "dask>=2022.02.1", + "distributed>=2022.02.1", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.4.0dev0",