diff --git a/.gitignore b/.gitignore index 045470926e..d338784258 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,8 @@ log dask-worker-space/ tmp/ .hypothesis +wheels/ +_skbuild/ ## files pickled in notebook when ran during python docstring generation docs/source/*.model diff --git a/build.sh b/build.sh index 9cd3e358b7..1c54276aa5 100755 --- a/build.sh +++ b/build.sh @@ -229,7 +229,6 @@ if (( ${CLEAN} == 1 )); then cd ${REPODIR} fi -# Before ################################################################################ # Configure for building all C++ targets @@ -283,11 +282,16 @@ fi # Build and (optionally) install the cuml Python package if completeBuild || hasArg cuml || hasArg pydocs; then + # Append `-DFIND_CUML_CPP=ON` to CUML_EXTRA_CMAKE_ARGS unless a user specified the option. + if [[ "${CUML_EXTRA_CMAKE_ARGS}" != *"DFIND_CUML_CPP"* ]]; then + CUML_EXTRA_CMAKE_ARGS="${CUML_EXTRA_CMAKE_ARGS} -DFIND_CUML_CPP=ON" + fi + cd ${REPODIR}/python + + python setup.py build_ext --inplace -- -DCMAKE_LIBRARY_PATH=${LIBCUML_BUILD_DIR} -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} ${CUML_EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} if [[ ${INSTALL_TARGET} != "" ]]; then - python setup.py build_ext -j${PARALLEL_LEVEL:-1} ${CUML_EXTRA_PYTHON_ARGS} --library-dir=${LIBCUML_BUILD_DIR} install --single-version-externally-managed --record=record.txt - else - python setup.py build_ext -j${PARALLEL_LEVEL:-1} ${CUML_EXTRA_PYTHON_ARGS} --library-dir=${LIBCUML_BUILD_DIR} + python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_LIBRARY_PATH=${LIBCUML_BUILD_DIR} -DCMAKE_MESSAGE_LOG_LEVEL=${CMAKE_LOG_LEVEL} ${CUML_EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} fi if hasArg pydocs; then diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index bca46849c5..5d36f17bcd 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -191,8 +191,8 @@ else gpuci_logger "Building and installing cuml" export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" export VERSION_SUFFIX="" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON} - gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} -c ${CONDA_BLD_DIR} cuml + gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cuml -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON} + gpuci_mamba_retry install cuml -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" gpuci_logger "Install the main version of dask, distributed, and dask-glm" set -x diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 401d8e0568..8722741f16 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -32,7 +32,10 @@ function sed_runner() { } sed_runner 's/'"CUML VERSION .* LANGUAGES"'/'"CUML VERSION ${NEXT_FULL_TAG} LANGUAGES"'/g' cpp/CMakeLists.txt -sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' cpp/CMakeLists.txt +sed_runner 's/'"set(CUML_VERSION .*)"'/'"set(CUML_VERSION ${NEXT_FULL_TAG})"'/g' python/CMakeLists.txt +# rapids-cmake version +sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake + # RTD update sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py diff --git a/conda/environments/cuml_dev_cuda11.0.yml b/conda/environments/cuml_dev_cuda11.0.yml index 489975876d..3cce856ffd 100644 --- a/conda/environments/cuml_dev_cuda11.0.yml +++ b/conda/environments/cuml_dev_cuda11.0.yml @@ -10,6 +10,7 @@ dependencies: - rapids-build-env=22.08.* - rapids-notebook-env=22.08.* - rapids-doc-env=22.08.* +- scikit-build>=0.13.1 - cudf=22.08.* - rmm=22.08.* - libcumlprims=22.08.* diff --git a/conda/environments/cuml_dev_cuda11.2.yml b/conda/environments/cuml_dev_cuda11.2.yml index 0332442b3a..998ec7de06 100644 --- a/conda/environments/cuml_dev_cuda11.2.yml +++ b/conda/environments/cuml_dev_cuda11.2.yml @@ -10,6 +10,7 @@ dependencies: - rapids-build-env=22.08.* - rapids-notebook-env=22.08.* - rapids-doc-env=22.08.* +- scikit-build>=0.13.1 - cudf=22.08.* - rmm=22.08.* - libcumlprims=22.08.* diff --git a/conda/environments/cuml_dev_cuda11.4.yml b/conda/environments/cuml_dev_cuda11.4.yml index b65ad25366..b3d1220859 100644 --- a/conda/environments/cuml_dev_cuda11.4.yml +++ b/conda/environments/cuml_dev_cuda11.4.yml @@ -10,6 +10,7 @@ dependencies: - rapids-build-env=22.08.* - rapids-notebook-env=22.08.* - rapids-doc-env=22.08.* +- scikit-build>=0.13.1 - cudf=22.08.* - rmm=22.08.* - libcumlprims=22.08.* diff --git a/conda/environments/cuml_dev_cuda11.5.yml b/conda/environments/cuml_dev_cuda11.5.yml index 32b3df3624..bf11ee9534 100644 --- a/conda/environments/cuml_dev_cuda11.5.yml +++ b/conda/environments/cuml_dev_cuda11.5.yml @@ -10,6 +10,7 @@ dependencies: - rapids-build-env=22.08.* - rapids-notebook-env=22.08.* - rapids-doc-env=22.08.* +- scikit-build>=0.13.1 - cudf=22.08.* - rmm=22.08.* - libcumlprims=22.08.* diff --git a/conda/recipes/cuml/build.sh b/conda/recipes/cuml/build.sh index 4c66e64f98..bfec44be2e 100644 --- a/conda/recipes/cuml/build.sh +++ b/conda/recipes/cuml/build.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Copyright (c) 2018-2022, NVIDIA CORPORATION. # This assumes the script is executed from the root of the repo directory -./build.sh cuml +./build.sh cuml -v diff --git a/conda/recipes/cuml/conda_build_config.yaml b/conda/recipes/cuml/conda_build_config.yaml index 322fe6faac..8db7dbb792 100644 --- a/conda/recipes/cuml/conda_build_config.yaml +++ b/conda/recipes/cuml/conda_build_config.yaml @@ -7,5 +7,8 @@ cxx_compiler_version: cuda_compiler: - nvcc +cmake_version: + - ">=3.20.1,!=3.23.0" + sysroot_version: - "2.17" diff --git a/conda/recipes/cuml/meta.yaml b/conda/recipes/cuml/meta.yaml index abaa4701c2..a1285a9146 100644 --- a/conda/recipes/cuml/meta.yaml +++ b/conda/recipes/cuml/meta.yaml @@ -26,7 +26,7 @@ build: requirements: build: - - cmake>=3.20.1,!=3.23.0 + - cmake {{ cmake_version }} - {{ compiler('c') }} - {{ compiler('cxx') }} - {{ compiler('cuda') }} {{ cuda_version }} @@ -34,6 +34,7 @@ requirements: host: - python x.x - setuptools + - scikit-build>=0.13.1 - cython>=0.29,<0.30 - treelite=2.4.0 - cudf {{ minor_version }} diff --git a/conda/recipes/libcuml/build.sh b/conda/recipes/libcuml/build.sh index 34758d0af2..84a4e3da07 100644 --- a/conda/recipes/libcuml/build.sh +++ b/conda/recipes/libcuml/build.sh @@ -7,4 +7,4 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh -n clean libcuml prims -v --allgpuarch +./build.sh -n libcuml prims -v --allgpuarch diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index dc70faf113..c00f2cce14 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,9 +15,9 @@ #============================================================================= cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.08/RAPIDS.cmake - ${CMAKE_BINARY_DIR}/RAPIDS.cmake) -include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) + +include(../fetch_rapids.cmake) + include(rapids-cmake) include(rapids-cpm) include(rapids-cuda) @@ -72,36 +72,36 @@ option(CUML_USE_RAFT_STATIC "Build and statically link the RAFT libraries" OFF) option(CUML_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" OFF) option(CUML_USE_TREELITE_STATIC "Build and statically link the treelite library" OFF) -message(VERBOSE "CUML: Building libcuml_c shared library. Contains the cuML C API: ${BUILD_CUML_C_LIBRARY}") -message(VERBOSE "CUML: Building libcuml shared library: ${BUILD_CUML_CPP_LIBRARY}") -message(VERBOSE "CUML: Building cuML algorithm tests: ${BUILD_CUML_TESTS}") -message(VERBOSE "CUML: Building cuML multigpu algorithm tests: ${BUILD_CUML_MG_TESTS}") -message(VERBOSE "CUML: Building ml-prims tests: ${BUILD_PRIMS_TESTS}") -message(VERBOSE "CUML: Building C++ API usage examples: ${BUILD_CUML_EXAMPLES}") -message(VERBOSE "CUML: Building cuML C++ benchmark tests: ${BUILD_CUML_BENCH}") -message(VERBOSE "CUML: Building ml-prims C++ benchmark tests: ${BUILD_CUML_PRIMS_BENCH}") -message(VERBOSE "CUML: Building the standard NCCL+UCX Communicator: ${BUILD_CUML_STD_COMMS}") -message(VERBOSE "CUML: Building the MPI+NCCL Communicator (used for testing): ${BUILD_CUML_MPI_COMMS}") -message(VERBOSE "CUML: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") -message(VERBOSE "CUML: Disabling OpenMP: ${DISABLE_OPENMP}") -message(VERBOSE "CUML: Enabling algorithms that use libcumlprims_mg: ${ENABLE_CUMLPRIMS_MG}") -message(VERBOSE "CUML: Enabling kernel resource usage info: ${KERNEL_INFO}") -message(VERBOSE "CUML: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") -message(VERBOSE "CUML: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") -message(VERBOSE "CUML: Enabling nvtx markers: ${NVTX}") -message(VERBOSE "CUML: Disabling all mnmg components and comms libraries: ${SINGLEGPU}") -message(VERBOSE "CUML: Cache build artifacts with ccache: ${USE_CCACHE}") -message(VERBOSE "CUML: Build and statically link RAFT libraries: ${CUML_USE_RAFT_STATIC}") -message(VERBOSE "CUML: Build and statically link FAISS library: ${CUML_USE_FAISS_STATIC}") -message(VERBOSE "CUML: Build and statically link Treelite library: ${CUML_USE_TREELITE_STATIC}") +message(VERBOSE "CUML_CPP: Building libcuml_c shared library. Contains the cuML C API: ${BUILD_CUML_C_LIBRARY}") +message(VERBOSE "CUML_CPP: Building libcuml shared library: ${BUILD_CUML_CPP_LIBRARY}") +message(VERBOSE "CUML_CPP: Building cuML algorithm tests: ${BUILD_CUML_TESTS}") +message(VERBOSE "CUML_CPP: Building cuML multigpu algorithm tests: ${BUILD_CUML_MG_TESTS}") +message(VERBOSE "CUML_CPP: Building ml-prims tests: ${BUILD_PRIMS_TESTS}") +message(VERBOSE "CUML_CPP: Building C++ API usage examples: ${BUILD_CUML_EXAMPLES}") +message(VERBOSE "CUML_CPP: Building cuML C++ benchmark tests: ${BUILD_CUML_BENCH}") +message(VERBOSE "CUML_CPP: Building ml-prims C++ benchmark tests: ${BUILD_CUML_PRIMS_BENCH}") +message(VERBOSE "CUML_CPP: Building the standard NCCL+UCX Communicator: ${BUILD_CUML_STD_COMMS}") +message(VERBOSE "CUML_CPP: Building the MPI+NCCL Communicator (used for testing): ${BUILD_CUML_MPI_COMMS}") +message(VERBOSE "CUML_CPP: Enabling detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") +message(VERBOSE "CUML_CPP: Disabling OpenMP: ${DISABLE_OPENMP}") +message(VERBOSE "CUML_CPP: Enabling algorithms that use libcumlprims_mg: ${ENABLE_CUMLPRIMS_MG}") +message(VERBOSE "CUML_CPP: Enabling kernel resource usage info: ${KERNEL_INFO}") +message(VERBOSE "CUML_CPP: Enabling kernelinfo in nvcc: ${CUDA_ENABLE_KERNEL_INFO}") +message(VERBOSE "CUML_CPP: Enabling lineinfo in nvcc: ${CUDA_ENABLE_LINE_INFO}") +message(VERBOSE "CUML_CPP: Enabling nvtx markers: ${NVTX}") +message(VERBOSE "CUML_CPP: Disabling all mnmg components and comms libraries: ${SINGLEGPU}") +message(VERBOSE "CUML_CPP: Cache build artifacts with ccache: ${USE_CCACHE}") +message(VERBOSE "CUML_CPP: Build and statically link RAFT libraries: ${CUML_USE_RAFT_STATIC}") +message(VERBOSE "CUML_CPP: Build and statically link FAISS library: ${CUML_USE_FAISS_STATIC}") +message(VERBOSE "CUML_CPP: Build and statically link Treelite library: ${CUML_USE_TREELITE_STATIC}") set(CUML_ALGORITHMS "ALL" CACHE STRING "Experimental: Choose which algorithms are built into libcuml++.so. Can specify individual algorithms or groups in a semicolon-separated list.") -message(VERBOSE "CUML: Building libcuml++ with algoriths: '${CUML_ALGORITHMS}'.") +message(VERBOSE "CUML_CPP: Building libcuml++ with algoriths: '${CUML_ALGORITHMS}'.") # Set RMM logging level set(RMM_LOGGING_LEVEL "INFO" CACHE STRING "Choose the logging level.") set_property(CACHE RMM_LOGGING_LEVEL PROPERTY STRINGS "TRACE" "DEBUG" "INFO" "WARN" "ERROR" "CRITICAL" "OFF") -message(VERBOSE "CUML: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") +message(VERBOSE "CUML_CPP: RMM_LOGGING_LEVEL = '${RMM_LOGGING_LEVEL}'.") ############################################################################## # - Target names ------------------------------------------------------------- @@ -121,7 +121,7 @@ set(PRIMS_BENCH_TARGET "prims_benchmark") if(DETECT_CONDA_ENV) rapids_cmake_support_conda_env( conda_env MODIFY_PREFIX_PATH ) if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT AND DEFINED ENV{CONDA_PREFIX}) - message(STATUS "CUML: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") + message(STATUS "CUML_CPP: No CMAKE_INSTALL_PREFIX argument detected, setting to: $ENV{CONDA_PREFIX}") set(CMAKE_INSTALL_PREFIX "$ENV{CONDA_PREFIX}") endif() endif() @@ -132,7 +132,7 @@ endif() if (NOT DISABLE_OPENMP) find_package(OpenMP) if(OpenMP_FOUND) - message(STATUS "CUML: OpenMP found in ${OPENMP_INCLUDE_DIRS}") + message(STATUS "CUML_CPP: OpenMP found in ${OPENMP_INCLUDE_DIRS}") endif() endif() @@ -170,8 +170,8 @@ endif() # SingleGPU build disables cumlprims_mg and comms components if(SINGLEGPU) - message(STATUS "Detected SINGLEGPU build option") - message(STATUS "Disabling Multi-GPU components and comms libraries") + message(STATUS "CUML_CPP: Detected SINGLEGPU build option") + message(STATUS "CUML_CPP: Disabling Multi-GPU components and comms libraries") set(BUILD_CUML_MG_TESTS OFF) set(BUILD_CUML_STD_COMMS OFF) set(BUILD_CUML_MPI_COMMS OFF) @@ -180,7 +180,7 @@ if(SINGLEGPU) endif() if(BUILD_CUML_MG_TESTS AND NOT SINGLEGPU) - message(STATUS "Detected BUILD_CUML_MG_TESTS set to ON. Enabling BUILD_CUML_MPI_COMMS") + message(STATUS "CUML_CPP: Detected BUILD_CUML_MG_TESTS set to ON. Enabling BUILD_CUML_MPI_COMMS") set(BUILD_CUML_MPI_COMMS ON) endif() diff --git a/fetch_rapids.cmake b/fetch_rapids.cmake new file mode 100644 index 0000000000..2b5c7e9d35 --- /dev/null +++ b/fetch_rapids.cmake @@ -0,0 +1,17 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.08/RAPIDS.cmake + ${CMAKE_BINARY_DIR}/RAPIDS.cmake +) +include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000000..849d6a0e57 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,134 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) + +include(../fetch_rapids.cmake) + +set(CUML_VERSION 22.08.00) + +project( + cuml-python + VERSION ${CUML_VERSION} + LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C + # language to be enabled here. The test project that is built in scikit-build to verify + # various linking options for the python library is hardcoded to build with C, so until + # that is fixed we need to keep C. + C CXX +) + +################################################################################ +# - User Options -------------------------------------------------------------- +option(FIND_CUML_CPP "Search for existing CUML C++ installations before defaulting to local files" OFF) +option(SINGLEGPU "Disable all mnmg components and comms libraries" OFF) + +# todo: use CMAKE_MESSAGE_CONTEXT for prefix for logging. +# https://github.com/rapidsai/cuml/issues/4843 +message(VERBOSE "CUML_PY: Searching for existing CUML C++ installations before defaulting to local files: ${FIND_CUML_CPP}") +message(VERBOSE "CUML_PY: Disabling all mnmg components and comms libraries: ${SINGLEGPU}") + +set(CUML_CPP_TARGET "cuml++") + +################################################################################ +# - Process User Options ------------------------------------------------------ + + +# If the user requested it, we attempt to find cuml. +if(FIND_CUML_CPP) + include(rapids-cpm) + rapids_cpm_init() + include(rapids-cuda) + rapids_cuda_init_architectures(cuml-python) + enable_language(CUDA) + + # variables used by get_raft.cmake, we can get rid of them when this is solved: + # https://github.com/rapidsai/rapids-cmake/issues/228 + set(CUML_VERSION_MAJOR "${cuml-python_VERSION_MAJOR}") + set(CUML_VERSION_MINOR "${cuml-python_VERSION_MINOR}") + set(CUML_USE_RAFT_NN ON) + set(CUML_USE_RAFT_DIST ON) + + # We need to call get_raft due to cuML asking for raft::nn and + # raft::distance targets + # see issue https://github.com/rapidsai/cuml/issues/4843 + include(../cpp/cmake/thirdparty/get_raft.cmake) + + # We need to call get_treelite explicitly because we need the correct + # ${TREELITE_LIBS} definition for RF + include(../cpp/cmake/thirdparty/get_treelite.cmake) + find_package(cuml ${CUML_VERSION} REQUIRED) + +else() + set(cuml_FOUND OFF) +endif() + + +if(NOT cuml_FOUND) + # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required + # languages for the C++ project even if this project does not require those languges. + include(rapids-cuda) + rapids_cuda_init_architectures(cuml-python) + enable_language(CUDA) + + # Since cuml only enables CUDA optionally, we need to manually include the file that + # rapids_cuda_init_architectures relies on `project` including. + include("${CMAKE_PROJECT_cuml-python_INCLUDE}") + + set(BUILD_CUML_TESTS OFF) + set(BUILD_PRIMS_TESTS OFF) + set(BUILD_CUML_C_LIBRARY OFF) + set(BUILD_CUML_EXAMPLES OFF) + set(BUILD_CUML_BENCH OFF) + set(BUILD_CUML_PRIMS_BENCH OFF) + message(STATUS "installing packages") + add_subdirectory(../cpp cuml-cpp) + + install(TARGETS ${CUML_CPP_TARGET} DESTINATION cuml/library) +endif() + +set(cuml_sg_libraries cuml::${CUML_CPP_TARGET}) + +if(NOT SINGLEGPU) + include(../cpp/cmake/thirdparty/get_cumlprims_mg.cmake) + set(cuml_mg_libraries + cuml::${CUML_CPP_TARGET} + cumlprims_mg::cumlprims_mg + ) +endif() + +include(rapids-cython) +rapids_cython_init() + +add_subdirectory(cuml/common) +add_subdirectory(cuml/internals) + +add_subdirectory(cuml/cluster) +add_subdirectory(cuml/datasets) +add_subdirectory(cuml/decomposition) +add_subdirectory(cuml/ensemble) +add_subdirectory(cuml/explainer) +add_subdirectory(cuml/fil) +add_subdirectory(cuml/kernel_ridge) +add_subdirectory(cuml/linear_model) +add_subdirectory(cuml/manifold) +add_subdirectory(cuml/metrics) +add_subdirectory(cuml/metrics/cluster) +add_subdirectory(cuml/neighbors) +add_subdirectory(cuml/random_projection) +add_subdirectory(cuml/solvers) +add_subdirectory(cuml/svm) +add_subdirectory(cuml/tsa) + +add_subdirectory(cuml/experimental/linear_model) + diff --git a/python/cuml/cluster/CMakeLists.txt b/python/cuml/cluster/CMakeLists.txt new file mode 100644 index 0000000000..3c39f06d34 --- /dev/null +++ b/python/cuml/cluster/CMakeLists.txt @@ -0,0 +1,40 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + agglomerative.pyx + dbscan.pyx + hdbscan.pyx + kmeans.pyx +) + +if(NOT SINGLEGPU) + list(APPEND cython_sources + dbscan_mg.pyx + kmeans_mg.pyx + ) +endif() + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES $,${cuml_sg_libraries},${cuml_mg_libraries}> + MODULE_PREFIX cluster_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/common/CMakeLists.txt b/python/cuml/common/CMakeLists.txt new file mode 100644 index 0000000000..41b6068dfc --- /dev/null +++ b/python/cuml/common/CMakeLists.txt @@ -0,0 +1,43 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +set(cython_sources + base.pyx + cuda.pyx + handle.pyx + logger.pyx + pointer_utils.pyx +) + +if(NOT SINGLEGPU) + list(APPEND cython_sources + opg_data_utils_mg.pyx + ) +endif() + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES $,${cuml_sg_libraries},${cuml_mg_libraries}> + MODULE_PREFIX common_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() + +# todo: ml_cuda_utils.h should be in the include folder of cuML or the functionality +# moved to another file, pointer_utils.pyx needs it +# https://github.com/rapidsai/cuml/issues/4841 +target_include_directories(common_pointer_utils PRIVATE "../../../cpp/src/") diff --git a/python/cuml/datasets/CMakeLists.txt b/python/cuml/datasets/CMakeLists.txt new file mode 100644 index 0000000000..eb2a37e2ac --- /dev/null +++ b/python/cuml/datasets/CMakeLists.txt @@ -0,0 +1,29 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +set(cython_sources + arima.pyx + regression.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX datasets_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/decomposition/CMakeLists.txt b/python/cuml/decomposition/CMakeLists.txt new file mode 100644 index 0000000000..1ab807ba7f --- /dev/null +++ b/python/cuml/decomposition/CMakeLists.txt @@ -0,0 +1,39 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + +set(cython_sources + pca.pyx + tsvd.pyx +) + +if(NOT SINGLEGPU) + list(APPEND cython_sources + base_mg.pyx + pca_mg.pyx + tsvd_mg.pyx + ) +endif() + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES $,${cuml_sg_libraries},${cuml_mg_libraries}> + MODULE_PREFIX decomposition_ + +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/ensemble/CMakeLists.txt b/python/cuml/ensemble/CMakeLists.txt new file mode 100644 index 0000000000..ac7f0184b1 --- /dev/null +++ b/python/cuml/ensemble/CMakeLists.txt @@ -0,0 +1,37 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + randomforest_common.pyx + randomforest_shared.pyx + randomforestclassifier.pyx + randomforestregressor.pyx +) + +set(linked_libraries + ${cuml_sg_libraries} + ${TREELITE_LIBS}) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${linked_libraries}" + MODULE_PREFIX ensemble_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/experimental/linear_model/CMakeLists.txt b/python/cuml/experimental/linear_model/CMakeLists.txt new file mode 100644 index 0000000000..e0988bb741 --- /dev/null +++ b/python/cuml/experimental/linear_model/CMakeLists.txt @@ -0,0 +1,30 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + lars.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX experimental_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../../library") +endforeach() diff --git a/python/cuml/explainer/CMakeLists.txt b/python/cuml/explainer/CMakeLists.txt new file mode 100644 index 0000000000..8a0a38c2bd --- /dev/null +++ b/python/cuml/explainer/CMakeLists.txt @@ -0,0 +1,33 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + base.pyx + kernel_shap.pyx + permutation_shap.pyx + tree_shap.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX explainer_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/fil/CMakeLists.txt b/python/cuml/fil/CMakeLists.txt new file mode 100644 index 0000000000..adea9765c0 --- /dev/null +++ b/python/cuml/fil/CMakeLists.txt @@ -0,0 +1,30 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + fil.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX fil_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/internals/CMakeLists.txt b/python/cuml/internals/CMakeLists.txt new file mode 100644 index 0000000000..7fa5f4bc5c --- /dev/null +++ b/python/cuml/internals/CMakeLists.txt @@ -0,0 +1,33 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + internals.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX internals_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() + +# We need to include for callbacks_implements.h in the internals folder +target_include_directories(internals_internals PRIVATE ${CMAKE_CURRENT_LIST_DIR}) diff --git a/python/cuml/kernel_ridge/CMakeLists.txt b/python/cuml/kernel_ridge/CMakeLists.txt new file mode 100644 index 0000000000..e4ca87d23d --- /dev/null +++ b/python/cuml/kernel_ridge/CMakeLists.txt @@ -0,0 +1,30 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + kernel_ridge.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX kernel_ridge_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/linear_model/CMakeLists.txt b/python/cuml/linear_model/CMakeLists.txt new file mode 100644 index 0000000000..ed25490a33 --- /dev/null +++ b/python/cuml/linear_model/CMakeLists.txt @@ -0,0 +1,44 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + +set(cython_sources + base.pyx + elastic_net.pyx + linear_regression.pyx + logistic_regression.pyx + mbsgd_classifier.pyx + mbsgd_regressor.pyx + ridge.pyx +) + +if(NOT SINGLEGPU) + list(APPEND cython_sources + base_mg.pyx + linear_regression_mg.pyx + ridge_mg.pyx + ) + +endif() + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES $,${cuml_sg_libraries},${cuml_mg_libraries}> + MODULE_PREFIX linear_model_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/manifold/CMakeLists.txt b/python/cuml/manifold/CMakeLists.txt new file mode 100644 index 0000000000..f4491936c9 --- /dev/null +++ b/python/cuml/manifold/CMakeLists.txt @@ -0,0 +1,33 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + simpl_set.pyx + t_sne.pyx + umap.pyx + umap_utils.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX manifold_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/metrics/CMakeLists.txt b/python/cuml/metrics/CMakeLists.txt new file mode 100644 index 0000000000..dcf72ece2e --- /dev/null +++ b/python/cuml/metrics/CMakeLists.txt @@ -0,0 +1,35 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + accuracy.pyx + hinge_loss.pyx + kl_divergence.pyx + pairwise_distances.pyx + regression.pyx + trustworthiness.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX metrics_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/metrics/cluster/CMakeLists.txt b/python/cuml/metrics/cluster/CMakeLists.txt new file mode 100644 index 0000000000..ebc57faf29 --- /dev/null +++ b/python/cuml/metrics/cluster/CMakeLists.txt @@ -0,0 +1,37 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + adjusted_rand_index.pyx + completeness_score.pyx + entropy.pyx + homogeneity_score.pyx + mutual_info_score.pyx + silhouette_score.pyx + utils.pyx + v_measure.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../../library") +endforeach() + diff --git a/python/cuml/neighbors/CMakeLists.txt b/python/cuml/neighbors/CMakeLists.txt new file mode 100644 index 0000000000..5065437149 --- /dev/null +++ b/python/cuml/neighbors/CMakeLists.txt @@ -0,0 +1,44 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + +set(cython_sources + ann.pyx + kneighbors_classifier.pyx + kneighbors_regressor.pyx + nearest_neighbors.pyx +) + +if(NOT SINGLEGPU) + list(APPEND cython_sources + kneighbors_classifier_mg.pyx + kneighbors_regressor_mg.pyx + nearest_neighbors_mg.pyx + ) +endif() + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES $,${cuml_sg_libraries},${cuml_mg_libraries}> + MODULE_PREFIX neighbors_ +) + +foreach(target IN LISTS targets_using_numpy) + target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}") +endforeach() + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/random_projection/CMakeLists.txt b/python/cuml/random_projection/CMakeLists.txt new file mode 100644 index 0000000000..12a87161db --- /dev/null +++ b/python/cuml/random_projection/CMakeLists.txt @@ -0,0 +1,30 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + random_projection.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX random_projection_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/solvers/CMakeLists.txt b/python/cuml/solvers/CMakeLists.txt new file mode 100644 index 0000000000..5867fcb83d --- /dev/null +++ b/python/cuml/solvers/CMakeLists.txt @@ -0,0 +1,38 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + cd.pyx + qn.pyx + sgd.pyx +) + +if(NOT SINGLEGPU) + list(APPEND cython_sources + cd_mg.pyx + ) +endif() + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES $,${cuml_sg_libraries},${cuml_mg_libraries}> + MODULE_PREFIX solvers_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/svm/CMakeLists.txt b/python/cuml/svm/CMakeLists.txt new file mode 100644 index 0000000000..865474ef22 --- /dev/null +++ b/python/cuml/svm/CMakeLists.txt @@ -0,0 +1,33 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + linear.pyx + svc.pyx + svm_base.pyx + svr.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX svm_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cuml/tests/dask/test_base.py b/python/cuml/tests/dask/test_dask_base.py similarity index 100% rename from python/cuml/tests/dask/test_base.py rename to python/cuml/tests/dask/test_dask_base.py diff --git a/python/cuml/tests/dask/test_coordinate_descent.py b/python/cuml/tests/dask/test_dask_coordinate_descent.py similarity index 100% rename from python/cuml/tests/dask/test_coordinate_descent.py rename to python/cuml/tests/dask/test_dask_coordinate_descent.py diff --git a/python/cuml/tests/dask/test_datasets.py b/python/cuml/tests/dask/test_dask_datasets.py similarity index 100% rename from python/cuml/tests/dask/test_datasets.py rename to python/cuml/tests/dask/test_dask_datasets.py diff --git a/python/cuml/tests/dask/test_dbscan.py b/python/cuml/tests/dask/test_dask_dbscan.py similarity index 100% rename from python/cuml/tests/dask/test_dbscan.py rename to python/cuml/tests/dask/test_dask_dbscan.py diff --git a/python/cuml/tests/dask/test_doctest.py b/python/cuml/tests/dask/test_dask_doctest.py similarity index 100% rename from python/cuml/tests/dask/test_doctest.py rename to python/cuml/tests/dask/test_dask_doctest.py diff --git a/python/cuml/tests/dask/test_func.py b/python/cuml/tests/dask/test_dask_func.py similarity index 100% rename from python/cuml/tests/dask/test_func.py rename to python/cuml/tests/dask/test_dask_func.py diff --git a/python/cuml/tests/dask/test_global_settings.py b/python/cuml/tests/dask/test_dask_global_settings.py similarity index 100% rename from python/cuml/tests/dask/test_global_settings.py rename to python/cuml/tests/dask/test_dask_global_settings.py diff --git a/python/cuml/tests/dask/test_input_utils.py b/python/cuml/tests/dask/test_dask_input_utils.py similarity index 100% rename from python/cuml/tests/dask/test_input_utils.py rename to python/cuml/tests/dask/test_dask_input_utils.py diff --git a/python/cuml/tests/dask/test_kmeans.py b/python/cuml/tests/dask/test_dask_kmeans.py similarity index 100% rename from python/cuml/tests/dask/test_kmeans.py rename to python/cuml/tests/dask/test_dask_kmeans.py diff --git a/python/cuml/tests/dask/test_kneighbors_classifier.py b/python/cuml/tests/dask/test_dask_kneighbors_classifier.py similarity index 100% rename from python/cuml/tests/dask/test_kneighbors_classifier.py rename to python/cuml/tests/dask/test_dask_kneighbors_classifier.py diff --git a/python/cuml/tests/dask/test_kneighbors_regressor.py b/python/cuml/tests/dask/test_dask_kneighbors_regressor.py similarity index 100% rename from python/cuml/tests/dask/test_kneighbors_regressor.py rename to python/cuml/tests/dask/test_dask_kneighbors_regressor.py diff --git a/python/cuml/tests/dask/test_label_binarizer.py b/python/cuml/tests/dask/test_dask_label_binarizer.py similarity index 100% rename from python/cuml/tests/dask/test_label_binarizer.py rename to python/cuml/tests/dask/test_dask_label_binarizer.py diff --git a/python/cuml/tests/dask/test_label_encoder.py b/python/cuml/tests/dask/test_dask_label_encoder.py similarity index 100% rename from python/cuml/tests/dask/test_label_encoder.py rename to python/cuml/tests/dask/test_dask_label_encoder.py diff --git a/python/cuml/tests/dask/test_linear_regression.py b/python/cuml/tests/dask/test_dask_linear_regression.py similarity index 100% rename from python/cuml/tests/dask/test_linear_regression.py rename to python/cuml/tests/dask/test_dask_linear_regression.py diff --git a/python/cuml/test/dask/test_logistic_regression.py b/python/cuml/tests/dask/test_dask_logistic_regression.py similarity index 100% rename from python/cuml/test/dask/test_logistic_regression.py rename to python/cuml/tests/dask/test_dask_logistic_regression.py diff --git a/python/cuml/tests/dask/test_metrics.py b/python/cuml/tests/dask/test_dask_metrics.py similarity index 100% rename from python/cuml/tests/dask/test_metrics.py rename to python/cuml/tests/dask/test_dask_metrics.py diff --git a/python/cuml/tests/dask/test_naive_bayes.py b/python/cuml/tests/dask/test_dask_naive_bayes.py similarity index 100% rename from python/cuml/tests/dask/test_naive_bayes.py rename to python/cuml/tests/dask/test_dask_naive_bayes.py diff --git a/python/cuml/tests/dask/test_nearest_neighbors.py b/python/cuml/tests/dask/test_dask_nearest_neighbors.py similarity index 100% rename from python/cuml/tests/dask/test_nearest_neighbors.py rename to python/cuml/tests/dask/test_dask_nearest_neighbors.py diff --git a/python/cuml/tests/dask/test_one_hot_encoder.py b/python/cuml/tests/dask/test_dask_one_hot_encoder.py similarity index 100% rename from python/cuml/tests/dask/test_one_hot_encoder.py rename to python/cuml/tests/dask/test_dask_one_hot_encoder.py diff --git a/python/cuml/tests/dask/test_pca.py b/python/cuml/tests/dask/test_dask_pca.py similarity index 100% rename from python/cuml/tests/dask/test_pca.py rename to python/cuml/tests/dask/test_dask_pca.py diff --git a/python/cuml/tests/dask/test_random_forest.py b/python/cuml/tests/dask/test_dask_random_forest.py similarity index 100% rename from python/cuml/tests/dask/test_random_forest.py rename to python/cuml/tests/dask/test_dask_random_forest.py diff --git a/python/cuml/tests/dask/test_ridge_regression.py b/python/cuml/tests/dask/test_dask_ridge_regression.py similarity index 100% rename from python/cuml/tests/dask/test_ridge_regression.py rename to python/cuml/tests/dask/test_dask_ridge_regression.py diff --git a/python/cuml/tests/dask/test_serialization.py b/python/cuml/tests/dask/test_dask_serialization.py similarity index 100% rename from python/cuml/tests/dask/test_serialization.py rename to python/cuml/tests/dask/test_dask_serialization.py diff --git a/python/cuml/tests/dask/test_tfidf.py b/python/cuml/tests/dask/test_dask_tfidf.py similarity index 100% rename from python/cuml/tests/dask/test_tfidf.py rename to python/cuml/tests/dask/test_dask_tfidf.py diff --git a/python/cuml/tests/dask/test_tsvd.py b/python/cuml/tests/dask/test_dask_tsvd.py similarity index 100% rename from python/cuml/tests/dask/test_tsvd.py rename to python/cuml/tests/dask/test_dask_tsvd.py diff --git a/python/cuml/tests/dask/test_umap.py b/python/cuml/tests/dask/test_dask_umap.py similarity index 100% rename from python/cuml/tests/dask/test_umap.py rename to python/cuml/tests/dask/test_dask_umap.py diff --git a/python/cuml/tsa/CMakeLists.txt b/python/cuml/tsa/CMakeLists.txt new file mode 100644 index 0000000000..756450a748 --- /dev/null +++ b/python/cuml/tsa/CMakeLists.txt @@ -0,0 +1,34 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + + + +set(cython_sources + arima.pyx + auto_arima.pyx + holtwinters.pyx + seasonality.pyx + stationarity.pyx +) + +rapids_cython_create_modules( + CXX + SOURCE_FILES "${cython_sources}" + LINKED_LIBRARIES "${cuml_sg_libraries}" + MODULE_PREFIX tsa_ +) + +foreach(cython_module IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + set_target_properties(${cython_module} PROPERTIES INSTALL_RPATH "\$ORIGIN;\$ORIGIN/../library") +endforeach() diff --git a/python/cython_build_ext.py b/python/cython_build_ext.py deleted file mode 100644 index 0c0cb5aeb1..0000000000 --- a/python/cython_build_ext.py +++ /dev/null @@ -1,250 +0,0 @@ -# -# Copyright (c) 2020, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys - -# TODO: It should be possible to support Cython-less distribution following -# this guide and removing the direct import of Cython: -# https://cython.readthedocs.io/en/latest/src/userguide/source_files_and_compilation.html#distributing-cython-modules - -# Must import in this order: -# setuptools -> Cython.Distutils.build_ext -> setuptools.command.build_ext -# Otherwise, setuptools.command.build_ext ends up inheriting from -# Cython.Distutils.old_build_ext which we do not want -import setuptools - -try: - from Cython.Distutils.build_ext import new_build_ext as _build_ext -except ImportError: - from setuptools.command.build_ext import build_ext as _build_ext - -import setuptools.command.build_ext - - -class cython_build_ext(_build_ext, object): - """ - This class follows the design of `Cython.Distutils.build_ext.new_build_ext` - to allow for parallel `cythonize()` but adds options for the various - arguments that can be passed to `cythonize()` including separate options - for `compiler_directives`. This build extension can be directly used in - place of `new_build_ext` for any Cython project that needs to set global - parameters in the build phase. See the documentation for more information - on the `cythonize()` arguments. - - Parameters - ---------- - annotate : bool, default=False - If True, will produce a HTML file for each of the .pyx or .py files - compiled. The HTML file gives an indication of how much Python - interaction there is in each of the source code lines, compared to - plain C code. It also allows you to see the C/C++ code generated for - each line of Cython code. This report is invaluable when optimizing a - function for speed, and for determining when to release the GIL: in - general, a nogil block may contain only “white” code. See examples in - Determining where to add types or Primes. - binding : bool, default=True - Controls whether free functions behave more like Python’s CFunctions - (e.g. len()) or, when set to True, more like Python’s functions. When - enabled, functions will bind to an instance when looked up as a class - attribute (hence the name) and will emulate the attributes of Python - functions, including introspections like argument names and - annotations. - - Changed in version 3.0.0: Default changed from False to True - cython_exclude : list of str - When passing glob patterns as module_list, you can exclude certain - module names explicitly by passing them into the exclude option. - embedsignature : bool, default=False - If set to True, Cython will embed a textual copy of the call signature - in the docstring of all Python visible functions and classes. Tools - like IPython and epydoc can thus display the signature, which cannot - otherwise be retrieved after compilation. - gdb_debug : bool, default=False - Passes the `gdb_debug` argument to `cythonize()`. Setting up debugging - for Cython can be difficult. See the debugging docs here - https://cython.readthedocs.io/en/latest/src/userguide/debugging.html - language_level : {"2", "3", "3str"}, default="2" - Globally set the Python language level to be used for module - compilation. Default is compatibility with Python 2. To enable Python 3 - source code semantics, set this to 3 (or 3str) - linetrace : bool, default=False - Write line tracing hooks for Python profilers or coverage reporting - into the compiled C code. This also enables profiling. Default is - False. Note that the generated module will not actually use line - tracing, unless you additionally pass the C macro definition - ``CYTHON_TRACE=1`` to the C compiler (e.g. using the setuptools option - define_macros). Define ``CYTHON_TRACE_NOGIL=1`` to also include nogil - functions and sections. - profile : bool, default=False - Write hooks for Python profilers into the compiled C code. - """ - user_options = [ - ("annotate=", - None, - "Passes the `annotate` argument to `cythonize()`. See the Cython " - "docs for more info."), - ("binding", - None, - "Sets the binding Cython compiler directive. See the Cython docs for " - "more info."), - ("cython-exclude=", - None, - "Sets the exclude argument for `cythonize()`. See the Cython docs for" - " more info."), - ("embedsignature", - None, - "Sets the `embedsignature` Cython compiler directive. See the Cython " - "docs for more info."), - ("gdb-debug=", - None, - "Passes the `gdb_debug` argument to `cythonize()`. See the Cython " - "docs for more info."), - ('language-level=', - None, - 'Sets the python language syntax to use "2", "3", "3str".'), - ("linetrace=", - None, - "Passes the `linetrace` argument to `cythonize()`. See the Cython " - "docs for more info."), - ("profile", - None, - "Sets the profile Cython compiler directive. See the Cython docs for " - "more info."), - ] + _build_ext.user_options - - boolean_options = [ - "annotate", - "binding", - "embedsignature", - "gdb-debug", - "linetrace", - "profile", - ] + _build_ext.boolean_options - - def initialize_options(self): - """ - Set the default values for the `user_options` to None to allow us to - detect if they were set by the user - """ - - self.annotate = None - self.binding = None - self.cython_exclude = None - self.embedsignature = None - self.gdb_debug = None - self.language_level = None - self.linetrace = None - self.profile = None - - super().initialize_options() - - def finalize_options(self): - """ - Determines any user defined options and finalizes the Cython - configuration before compilation - """ - - # Ensure the base build class options get set so we can use parallel - self.set_undefined_options( - 'build', - ('build_lib', 'build_lib'), - ('build_temp', 'build_temp'), - ('compiler', 'compiler'), - ('debug', 'debug'), - ('force', 'force'), - ('parallel', 'parallel'), - ('plat_name', 'plat_name'), - ) - - # If ext_modules is set, then build the cythonize argument list - if self.distribution.ext_modules: - if self.language_level is None: - self.language_level = str(sys.version_info[0]) - - assert self.language_level in ( - '2', '3', - '3str'), 'Incorrect Cython language level ("{0}")'.format( - self.language_level) - - compiler_directives = dict(language_level=self.language_level) - - if (self.binding is not None): - self.binding = bool(self.binding) - compiler_directives.update({"binding": self.binding}) - - if (self.profile is not None): - self.profile = bool(self.profile) - compiler_directives.update({"profile": self.profile}) - - if (self.linetrace is not None): - self.linetrace = bool(self.linetrace) - compiler_directives.update({"linetrace": self.linetrace}) - - # Also need the compiler directive. Only add if it hasnt been - # specified yet - for ext in self.distribution.ext_modules: - if (not hasattr(ext, "define_macros")): - ext.define_macros = [] - - found_macro = False - - for mac in ext.define_macros: - if (mac[0] == "CYTHON_TRACE_NOGIL"): - found_macro = True - break - - if not found_macro: - ext.define_macros.append(("CYTHON_TRACE_NOGIL", 1)) - - if (self.embedsignature is not None): - self.embedsignature = bool(self.embedsignature) - compiler_directives.update( - {"embedsignature": self.embedsignature}) - - cythonize_kwargs = {} - - if (self.annotate is not None): - - cythonize_kwargs.update({"annotate": self.annotate}) - - if (self.cython_exclude is not None): - - if (isinstance(self.cython_exclude, str)): - self.cython_exclude = list(self.cython_exclude) - - cythonize_kwargs.update({"exclude": self.cython_exclude}) - - if (self.gdb_debug is not None): - - cythonize_kwargs.update({"gdb_debug": self.gdb_debug}) - - # Handle nthreads separately to mimic what Cython does - nthreads = getattr(self, 'parallel', None) # -j option in Py3.5+ - nthreads = int(nthreads) if nthreads else None - - # Delay import this to allow for Cython-less installs - from Cython.Build.Dependencies import cythonize - - # Finally, cythonize the arguments - self.distribution.ext_modules = cythonize( - self.distribution.ext_modules, - nthreads=nthreads, - force=self.force, - compiler_directives=compiler_directives, - **cythonize_kwargs) - - # Skip calling super() and jump straight to setuptools - setuptools.command.build_ext.build_ext.finalize_options(self) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000000..e3fe544bef --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,24 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] + +requires = [ + "wheel", + "setuptools", + "cython>=0.29,<0.30", + "scikit-build>=0.13.1", + "cmake>=3.20.1,!=3.23.0", + "ninja", +] diff --git a/python/pytest.ini b/python/pytest.ini index b6b48b8ce0..6dec495c6b 100644 --- a/python/pytest.ini +++ b/python/pytest.ini @@ -7,7 +7,11 @@ markers = memleak: Test that checks for memory leaks no_bad_cuml_array_check: Test that should not check for bad CumlArray uses -testpaths = cuml/tests +testpaths = + cuml/tests + cuml/tests/dask + cuml/tests/explainer + cuml/tests/stemmer_tests filterwarnings = error::FutureWarning:cuml[.*] # Catch uses of deprecated positional args in testing diff --git a/python/setup.cfg b/python/setup.cfg index 0118468db4..b9758e4886 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -1,8 +1,8 @@ -# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. [flake8] filename = *.py, *.pyx, *.pxd -exclude = +exclude = cpp, thirdparty, versioneer.py, @@ -20,7 +20,7 @@ exclude = # W503: line break before binary operator (breaks lines that start with a pointer) # W504: line break after binary operator (breaks lines that end with a pointer) -per-file-ignores = +per-file-ignores = # imported but unused __init__.py: F401 # Cython Exclusions @@ -38,12 +38,18 @@ versionfile_build = cuml/_version.py tag_prefix = v parentdir_prefix = cuml- +[pytest] +markers = + unit: Quickest tests focused on accuracy and correctness + quality: More intense tests than unit with increased runtimes + stress: Longest running tests focused on stressing hardware compute resources + mg: Multi-GPU tests + memleak: Test that checks for memory leaks + no_bad_cuml_array_check: Test that should not check for bad CumlArray uses -# Project wide, Cython settings -[build_ext] -inplace = True -binding = True -language_level = 3 -profile = False -linetrace = False -embedsignature = True \ No newline at end of file +testpaths = cuml/tests + +filterwarnings = + error::FutureWarning:cuml[.*] # Catch uses of deprecated positional args in testing + ignore:[^.]*ABCs[^.]*:DeprecationWarning:patsy[.*] + ignore:(.*)alias(.*):DeprecationWarning:hdbscan[.*] diff --git a/python/setup.py b/python/setup.py index 023a9f47de..b8a4b2e7ee 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,29 +14,19 @@ # limitations under the License. # -import glob import os import shutil import sys -import sysconfig -import warnings -from pprint import pprint from pathlib import Path from setuptools import find_packages -from setuptools import setup -from setuptools.extension import Extension -from distutils.sysconfig import get_python_lib -from distutils.command.build import build as _build - -import numpy from setuputils import clean_folder from setuputils import get_environment_option from setuputils import get_cli_option import versioneer -from cython_build_ext import cython_build_ext +from skbuild import setup install_requires = ['numba', 'cython'] @@ -81,7 +71,9 @@ shutil.rmtree(setup_file_path + '/__pycache__', ignore_errors=True) clean_folder(setup_file_path + '/cuml') - shutil.rmtree(setup_file_path + '/build') + shutil.rmtree(setup_file_path + '/build', ignore_errors=True) + shutil.rmtree(setup_file_path + '/_skbuild', ignore_errors=True) + shutil.rmtree(setup_file_path + '/dist', ignore_errors=True) except IOError: pass @@ -97,167 +89,10 @@ sys.exit(0) -if "--multigpu" in sys.argv: - warnings.warn("Flag --multigpu is deprecated. By default cuML is" - "built with multi GPU support. To disable it use the flag" - "--singlegpu") - sys.argv.remove('--multigpu') - if not libcuml_path: libcuml_path = '../cpp/build/' -############################################################################## -# - Cython extensions build and parameters ----------------------------------- -# -# We create custom build steps for both `build` and `build_ext` for several -# reasons: -# 1) Custom `build_ext` is needed to set `cython_build_ext.cython_exclude` when -# `--singlegpu=True` -# 2) Custom `build` is needed to exclude pacakges and directories when -# `--singlegpu=True` -# 3) These cannot be combined because `build` is used by both `build_ext` and -# `install` commands and it would be difficult to set -# `cython_build_ext.cython_exclude` from `cuml_build` since the property -# exists on a different command. -# -# Using custom commands also allows combining commands at the command line. For -# example, the following will all work as expected: -# `python setup.py clean --all build --singlegpu build_ext --inplace` -# `python setup.py clean --all build --singlegpu install --record=record.txt` -# `python setup.py build_ext --debug --singlegpu` - - -class cuml_build(_build): - - def initialize_options(self): - - self.singlegpu = False - super().initialize_options() - - def finalize_options(self): - - # distutils plain build command override cannot be done just setting - # user_options and boolean options like build_ext below. Distribution - # object has all the args used by the user, we can check that. - self.singlegpu = '--singlegpu' in self.distribution.script_args - - libs = ['cuml++', 'cudart', 'cusparse', 'cusolver'] - - include_dirs = [ - '../cpp/src', - '../cpp/include', - '../cpp/src_prims', - cuda_include_dir, - numpy.get_include(), - '../cpp/build/faiss/src/faiss', - os.path.dirname(sysconfig.get_path("include")) - ] - - python_exc_list = [] - - if (self.singlegpu): - python_exc_list = ["*.dask", "*.dask.*"] - else: - libs.append('cumlprims_mg') - libs.append('nccl') - - sys_include = os.path.dirname(sysconfig.get_path("include")) - include_dirs.append("%s/cumlprims" % sys_include) - - # Find packages now that --singlegpu has been determined - self.distribution.packages = find_packages(include=['cuml', 'cuml.*'], - exclude=python_exc_list) - - # Build the extensions list - extensions = [ - Extension("*", - sources=["cuml/**/*.pyx"], - include_dirs=include_dirs, - library_dirs=[ - get_python_lib(), - libcuml_path, - cuda_lib_dir, - os.path.join(os.sys.prefix, "lib") - ], - libraries=libs, - language='c++', - extra_compile_args=['-std=c++17']) - ] - - self.distribution.ext_modules = extensions - - super().finalize_options() - - -# This custom build_ext is only responsible for setting cython_exclude when -# --singlegpu is specified -class cuml_build_ext(cython_build_ext, object): - user_options = [ - ("singlegpu", None, "Specifies whether to include multi-gpu or not"), - ] + cython_build_ext.user_options - - boolean_options = ["singlegpu"] + cython_build_ext.boolean_options - - def build_extensions(self): - def remove_flags(compiler, *flags): - for flag in flags: - try: - compiler.compiler_so = list( - filter((flag).__ne__, compiler.compiler_so) - ) - except Exception: - pass - # Full optimization - self.compiler.compiler_so.append("-O3") - - # Ignore deprecation declaraction warnings - self.compiler.compiler_so.append("-Wno-deprecated-declarations") - - # adding flags to always add symbols/link of libcuml++ and transitive - # dependencies to Cython extensions - self.compiler.linker_so.append("-Wl,--no-as-needed") - - # No debug symbols, full optimization, no '-Wstrict-prototypes' warning - remove_flags( - self.compiler, "-g", "-G", "-O1", "-O2", "-Wstrict-prototypes" - ) - cython_build_ext.build_extensions(self) - - def initialize_options(self): - - self.singlegpu = None - - super().initialize_options() - - def finalize_options(self): - - # Ensure the base build class options get set so we can use singlegpu - self.set_undefined_options( - 'build', - ('singlegpu', 'singlegpu'), - ) - - # Exclude multigpu components that use libcumlprims if - # --singlegpu is used - if (self.singlegpu): - cython_exc_list = glob.glob('cuml/*/*_mg.pyx') - cython_exc_list = cython_exc_list + glob.glob('cuml/*/*_mg.pxd') - - print('--singlegpu: excluding the following Cython components:') - pprint(cython_exc_list) - - # Append to base excludes - self.cython_exclude = cython_exc_list + \ - (self.cython_exclude or []) - - super().finalize_options() - - -# Specify the custom build class -cmdclass = dict() -cmdclass.update(versioneer.get_cmdclass()) -cmdclass["build"] = cuml_build -cmdclass["build_ext"] = cuml_build_ext +cmdclass = versioneer.get_cmdclass() ############################################################################## # - Python package generation ------------------------------------------------ @@ -272,7 +107,12 @@ def finalize_options(self): "Programming Language :: Python :: 3.9" ], author="NVIDIA Corporation", - setup_requires=['cython'], + url="https://github.com/rapidsai/cudf", + setup_requires=['Cython>=0.29,<0.30'], + packages=find_packages(include=['cuml', 'cuml.*']), + package_data={ + key: ["*.pxd"] for key in find_packages(include=['cuml', 'cuml.*']) + }, install_requires=install_requires, license="Apache", cmdclass=cmdclass, diff --git a/python/setuputils.py b/python/setuputils.py index 3e8aa248f2..a8085802be 100644 --- a/python/setuputils.py +++ b/python/setuputils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,11 +18,8 @@ import os import re import shutil -import subprocess import sys -from pathlib import Path - def get_environment_option(name): env_variable = os.environ.get(name, False) @@ -69,221 +66,21 @@ def clean_folder(path): os.remove(file) -def use_raft_package(raft_path, cpp_build_path, - git_info_file=None): - """ - Function to use the python code in RAFT in package.raft - - - If RAFT symlink already exists, don't change anything. Use setup.py clean - if you want to change RAFT location. - - Uses RAFT located in $RAFT_PATH if $RAFT_PATH exists. - - Otherwise it will look for RAFT in the libcuml build folder, - located either in the default location ../cpp/build or in - $CUML_BUILD_PATH. - - Otherwise it will clone RAFT into _external_repositories. - - Branch/git tag cloned is located in git_info_file in this case. - - """ - if os.path.islink('cuml/raft'): - raft_path = os.path.realpath('cuml/raft') - # walk up two dirs from `python/raft` - raft_path = os.path.join(raft_path, '..', '..') - print("-- Using existing RAFT folder") - elif isinstance(raft_path, (str, os.PathLike)): - print('-- Using RAFT_PATH argument') - elif os.environ.get('RAFT_PATH', False) is not False: - raft_path = str(os.environ['RAFT_PATH']) - print('-- Using RAFT_PATH environment variable') - else: - raft_path, raft_cloned = \ - clone_repo_if_needed('raft', cpp_build_path, - git_info_file=git_info_file) - raft_path = os.path.join('../', raft_path) - - raft_path = os.path.realpath(raft_path) - print('-- RAFT found at: ' + str(raft_path)) - - try: - os.symlink( - os.path.join(raft_path, 'python/raft'), - os.path.join('cuml/raft') - ) - except FileExistsError: - os.remove(os.path.join('cuml/raft')) - os.symlink( - os.path.join(raft_path, 'python/raft'), - os.path.join('cuml/raft') - ) - - return os.path.join(raft_path, 'cpp/include') - - -def clone_repo_if_needed(name, cpp_build_path=None, - git_info_file=None): - if git_info_file is None: - git_info_file = \ - _get_repo_path() + '/cpp/cmake/thirdparty/get_raft.cmake' - - if cpp_build_path is None or cpp_build_path is False: - cpp_build_path = _get_repo_path() + '/cpp/build/_deps/' - - repo_cloned = get_submodule_dependency(name, - cpp_build_path=cpp_build_path, - git_info_file=git_info_file) - - if repo_cloned: - repo_path = ( - _get_repo_path() + '/python/_external_repositories/' + name + '/') - else: - repo_path = os.path.join(cpp_build_path, name + '-src/') - - return repo_path, repo_cloned - - -def get_submodule_dependency(repo, - git_info_file, - cpp_build_path): - """ - Function to check if sub repositories (i.e. submodules in git terminology) - already exist in the libcuml build folder, otherwise will clone the - repos needed to build the cuML Python package. - - Parameters - ---------- - repo : Strings or list of Strings - Name of the repos to be cloned. Must match - the names of the cmake git clone instruction - `ExternalProject_Add(name` - git_info_file : String - Relative path of the location of the CMakeLists.txt (or the cmake - module which contains ExternalProject_Add definitions) to extract - the information. By default it will look in the standard location - `cuml_repo_root/cpp` - cpp_build_path : String - Relative location of the build folder to look if repositories - already exist - - Returns - ------- - result : boolean - True if repos were found in libcuml cpp build folder, False - if they were not found. - """ - - if isinstance(repo, str): - repos = [repo] - - repo_info = get_repo_cmake_info(repos, git_info_file) - - if os.path.exists(os.path.join(cpp_build_path, repos[0] + '-src/')): - print("-- Third party modules found succesfully in the libcuml++ " - "build folder:") - print(" " + str(cpp_build_path)) - - return False - - else: - - print("-- Third party repositories have not been found so they " - "will be cloned. To avoid this set the environment " - "variable CUML_BUILD_PATH, containing the absolute " - "path to the build folder where libcuml++ was built. ") - - for repo in repos: - clone_repo(repo, repo_info[repo][0], repo_info[repo][1]) - - return True - - -def clone_repo(name, GIT_REPOSITORY, GIT_TAG, - location_to_clone='_external_repositories/', force_clone=False): - """ - Function to clone repos if they have not been cloned already. - Variables are named identical to the cmake counterparts for clarity, - in spite of not being very pythonic. - - Parameters - ---------- - name : String - Name of the repo to be cloned - GIT_REPOSITORY : String - URL of the repo to be cloned - GIT_TAG : String - commit hash or git hash to be cloned. Accepts anything that - `git checkout` accepts - force_clone : Boolean - Set to True to ignore already cloned repositories in - _external_repositories and clone - location_to_clone: String (default: '_external_repositories/') - Name that will contain clone of repository if needed. - """ - - if not os.path.exists(location_to_clone + name) or force_clone: - print("Cloning repository " + name + " into " + location_to_clone + - name) - subprocess.check_call(['git', 'clone', - GIT_REPOSITORY, - location_to_clone + name]) - wd = os.getcwd() - os.chdir(location_to_clone + name) - subprocess.check_call(['git', 'checkout', - GIT_TAG]) - os.chdir(wd) - else: - print("Found repository " + name + " in _external_repositories/" + - name) - - -def get_repo_cmake_info(names, file_path): - """ - Function to find information about submodules from cpp/CMakeLists file - - Parameters - ---------- - name : List of Strings - List containing the names of the repos to be cloned. Must match - the names of the cmake git clone instruction - `ExternalProject_Add(name` - file_path : String - Relative path of the location of the CMakeLists.txt (or the cmake - module which contains ExternalProject_Add definitions) to extract - the information. - - Returns - ------- - results : dictionary - Dictionary where results[name] contains an array, - where results[name][0] is the url of the repo and - repo_info[repo][1] is the tag/commit hash to be cloned as - specified by cmake. - - """ - with open(file_path) as f: - s = f.read() - - results = {} - - for name in names: - repo = re.findall(r'\s.*GIT_REPOSITORY.*', s) - repo = repo[-1].split()[-1] - fork = re.findall(r'\s.*FORK.*', s) - fork = fork[-1].split()[-1] - repo = repo.replace("${PKG_FORK}", fork) - tag = re.findall(r'\s.*PINNED_TAG.*', s) - tag = tag[-1].split()[-1] - if tag == 'branch-${CUML_BRANCH_VERSION_raft}': - loc = _get_repo_path() + '/cpp/CMakeLists.txt' - with open(loc) as f: - cmakelists = f.read() - tag = re.findall(r'\s.*project\(CUML VERSION.*', cmakelists) - tag = tag[-1].split()[2].split('.') - tag = 'branch-{}.{}'.format(tag[0], tag[1]) - - results[name] = [repo, tag] +def get_cuda_version_from_header(cuda_include_dir, delimiter=""): - return results + cuda_version = None + with open(os.path.join(cuda_include_dir, "cuda.h"), encoding="utf-8") as f: + for line in f.readlines(): + if re.search(r"#define CUDA_VERSION ", line) is not None: + cuda_version = line + break -def _get_repo_path(): - python_dir = Path(__file__).resolve() - return str(python_dir.parent.parent.absolute()) + if cuda_version is None: + raise TypeError("CUDA_VERSION not found in cuda.h") + cuda_version = int(cuda_version.split()[2]) + return "%d%s%d" % ( + cuda_version // 1000, + delimiter, + (cuda_version % 1000) // 10, + )