Merge pull request #5315 from rongou/fetch-thrust

[REVIEW] fetch thrust/cub from github
rapidsai · Jun 5, 2020 · efebcd1 · efebcd1
2 parents d3a4d2c + 7dd5b47
commit efebcd1
Show file tree

Hide file tree

Showing 13 changed files with 64 additions and 17 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,7 +1,3 @@
-[submodule "thirdparty/cub"]
-	path = thirdparty/cub
-	url = https://github.com/rapidsai/thirdparty-cub.git
-	branch = cudf-cub
 [submodule "thirdparty/jitify"]
 	path = thirdparty/jitify
 	url = https://github.com/rapidsai/jitify.git

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@
 ## Improvements
 
 - PR #5245 Add column reduction benchmark
+- PR #5315 Use CMake `FetchContent` to obtain latest release of `cub` and `thrust`
 - PR #5268 Rely on NumPy arrays for out-of-band pickling
 - PR #5288 Drop `auto_pickle` decorator #5288
 - PR #5231 Type `Buffer` as `uint8`

diff --git a/conda/environments/cudf_dev_cuda10.0.yml b/conda/environments/cudf_dev_cuda10.0.yml
@@ -10,7 +10,7 @@ dependencies:
   - clang-tools=8.0.1
   - cupy>=6.6.0,<8.0.0a0,!=7.1.0
   - rmm=0.15.*
-  - cmake>=3.12
+  - cmake>=3.14
   - cmake_setuptools>=0.1.3
   - python>=3.6,<3.8
   - numba>=0.49.0

diff --git a/conda/environments/cudf_dev_cuda10.1.yml b/conda/environments/cudf_dev_cuda10.1.yml
@@ -10,7 +10,7 @@ dependencies:
   - clang-tools=8.0.1
   - cupy>=6.6.0,<8.0.0a0,!=7.1.0
   - rmm=0.15.*
-  - cmake>=3.12
+  - cmake>=3.14
   - cmake_setuptools>=0.1.3
   - python>=3.6,<3.8
   - numba>=0.49.0

diff --git a/conda/environments/cudf_dev_cuda10.2.yml b/conda/environments/cudf_dev_cuda10.2.yml
@@ -10,7 +10,7 @@ dependencies:
   - clang-tools=8.0.1
   - cupy>=6.6.0,<8.0.0a0,!=7.1.0
   - rmm=0.15.*
-  - cmake>=3.12
+  - cmake>=3.14
   - cmake_setuptools>=0.1.3
   - python>=3.6,<3.8
   - numba>=0.49.0

diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml
@@ -25,7 +25,7 @@ build:
 
 requirements:
   build:
-    - cmake >=3.12.4
+    - cmake >=3.14.0
   host:
     - librmm {{ minor_version }}.*
     - cudatoolkit {{ cuda_version }}.*

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #=============================================================================
-cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 project(CUDA_DATAFRAME VERSION 0.15.0 LANGUAGES C CXX CUDA)
 
@@ -161,6 +161,12 @@ include(FeatureSummary)
 include(CheckIncludeFiles)
 include(CheckLibraryExists)
 
+###################################################################################################
+# - third party dependencies-----------------------------------------------------------------------
+# add third party dependencies using CMake FetchContent
+
+add_subdirectory(thirdparty)
+
 ###################################################################################################
 # - conda environment -----------------------------------------------------------------------------
 
@@ -325,6 +331,9 @@ message(STATUS "BENCHMARK_LIST set to: ${BENCHMARK_LIST}")
 ###################################################################################################
 # - include paths ---------------------------------------------------------------------------------
 
+include_directories("${CUB_INCLUDE_DIR}"
+                    "${THRUST_INCLUDE_DIR}")
+
 if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
 	include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
 endif(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
@@ -333,7 +342,6 @@ include_directories("${CMAKE_BINARY_DIR}/include"
                     "${CMAKE_BINARY_DIR}/include/jit"
                     "${CMAKE_SOURCE_DIR}/include"
                     "${CMAKE_SOURCE_DIR}/src"
-                    "${CMAKE_SOURCE_DIR}/thirdparty/cub"
                     "${CMAKE_SOURCE_DIR}/thirdparty/jitify"
                     "${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include"
                     "${ARROW_INCLUDE_DIR}"

diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 project(CUDF_BENCHS LANGUAGES C CXX CUDA)
 
@@ -22,12 +22,13 @@ endfunction(ConfigureBench)
 ###################################################################################################
 # - include paths ---------------------------------------------------------------------------------
 
-include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
+include_directories("${CUB_INCLUDE_DIR}"
+                    "${THRUST_INCLUDE_DIR}"
+                    "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
                     "${CMAKE_BINARY_DIR}/include"
                     "${CMAKE_SOURCE_DIR}/include"
                     "${CMAKE_SOURCE_DIR}"
                     "${CMAKE_SOURCE_DIR}/src"
-                    "${CMAKE_SOURCE_DIR}/thirdparty/cub"
                     "${CMAKE_SOURCE_DIR}/thirdparty/dlpack/include"
                     "${CMAKE_SOURCE_DIR}/thirdparty/jitify"
                     "${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include"

diff --git a/cpp/src/join/join_kernels.cuh b/cpp/src/join/join_kernels.cuh
@@ -203,7 +203,7 @@ __device__ void flush_output_cache(const unsigned int activemask,
 
   if (0 == lane_id) { output_offset = atomicAdd(current_idx, current_idx_shared[warp_id]); }
 
-  output_offset = cub::ShuffleIndex(output_offset, 0, detail::warp_size, activemask);
+  output_offset = cub::ShuffleIndex<detail::warp_size>(output_offset, 0, activemask);
 
   for (int shared_out_idx = lane_id; shared_out_idx < current_idx_shared[warp_id];
        shared_out_idx += num_threads) {

diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 project(CUDF_TESTS LANGUAGES C CXX CUDA)
 
@@ -58,6 +58,9 @@ endfunction(ConfigureTest)
 ###################################################################################################
 # - include paths ---------------------------------------------------------------------------------
 
+include_directories("${CUB_INCLUDE_DIR}"
+                    "${THRUST_INCLUDE_DIR}")
+
 if(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES)
     include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
 endif()
@@ -66,7 +69,6 @@ include_directories("${CMAKE_BINARY_DIR}/include"
                     "${CMAKE_SOURCE_DIR}/include"
                     "${CMAKE_SOURCE_DIR}"
                     "${CMAKE_SOURCE_DIR}/src"
-                    "${CMAKE_SOURCE_DIR}/thirdparty/cub"
                     "${CMAKE_SOURCE_DIR}/thirdparty/jitify"
                     "${CMAKE_SOURCE_DIR}/thirdparty/dlpack/include"
                     "${CMAKE_SOURCE_DIR}/thirdparty/libcudacxx/include"

diff --git a/python/cudf/setup.py b/python/cudf/setup.py
@@ -44,6 +44,8 @@
             "../../cpp/include/cudf",
             "../../cpp/include",
             "../../cpp/build/include",
+            "../../cpp/build/_deps/cub-src",
+            "../../cpp/build/_deps/thrust-src",
             "../../thirdparty/libcudacxx/include",
             os.path.dirname(sysconfig.get_path("include")),
             np.get_include(),

diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt
@@ -0,0 +1,38 @@
+include(FetchContent)
+
+###################################################################################################
+# - cub -------------------------------------------------------------------------------------------
+
+FetchContent_Declare(
+    cub
+    GIT_REPOSITORY https://github.com/thrust/cub.git
+    GIT_TAG        1.9.10
+    GIT_SHALLOW    true
+    # TODO(rongou): remove this after upgrading to new CUB release.
+    PATCH_COMMAND
+        COMMAND    sed -i "494 a histogram_sweep_config.template Init<typename Policy110::HistogramSweepPolicy>()$<SEMICOLON>" cub/device/dispatch/dispatch_histogram.cuh
+)
+
+FetchContent_GetProperties(cub)
+if(NOT cub_POPULATED)
+  FetchContent_Populate(cub)
+  # We are not using the cub CMake targets, so no need to call `add_subdirectory()`.
+endif()
+set(CUB_INCLUDE_DIR "${cub_SOURCE_DIR}" PARENT_SCOPE)
+
+###################################################################################################
+# - thrust ----------------------------------------------------------------------------------------
+
+FetchContent_Declare(
+    thrust
+    GIT_REPOSITORY https://github.com/thrust/thrust.git
+    GIT_TAG        1.9.10
+    GIT_SHALLOW    true
+)
+
+FetchContent_GetProperties(thrust)
+if(NOT thrust_POPULATED)
+  FetchContent_Populate(thrust)
+  # We are not using the thrust CMake targets, so no need to call `add_subdirectory()`.
+endif()
+set(THRUST_INCLUDE_DIR "${thrust_SOURCE_DIR}" PARENT_SCOPE)
diff --git a/thirdparty/cub b/thirdparty/cub