diff --git a/.gitignore b/.gitignore index e01e6ca3..e0e414ed 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,6 @@ docs/_build ## VSCode IDE .vscode + +# vim +*.sw[a-z] diff --git a/cmake-format-rapids-cmake.json b/cmake-format-rapids-cmake.json index 6d3b753f..6c50803f 100644 --- a/cmake-format-rapids-cmake.json +++ b/cmake-format-rapids-cmake.json @@ -236,6 +236,21 @@ "GLOBAL_TARGETS": "+", "FIND_ARGS": "+" } + }, + "rapids_cython_init": { + "pargs": { + "nargs": "0" + } + }, + "rapids_cython_create_modules": { + "pargs": { + "nargs": "0" + }, + "kwargs": { + "SOURCE_FILES": "*", + "LINKED_LIBRARIES": "*", + "INSTALL_DIR": "1" + } } } diff --git a/rapids-cmake/cython/create_modules.cmake b/rapids-cmake/cython/create_modules.cmake new file mode 100644 index 00000000..8ef5a9df --- /dev/null +++ b/rapids-cmake/cython/create_modules.cmake @@ -0,0 +1,96 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +include_guard(GLOBAL) + +#[=======================================================================[.rst: +rapids_cython_create_modules +---------------------------- + +.. versionadded:: v22.06.00 + +Generate C(++) from Cython and create Python modules. + +.. code-block:: cmake + + rapids_cython_create_modules([CXX] [SOURCE_FILES ...] [LINKED_LIBRARIES ... ] [INSTALL_DIR ) + +Creates a Cython target for a module, then adds a corresponding Python +extension module. + +.. note:: + Requires :cmake:command:`rapids_cython_init` to be called before usage. + +``CXX`` + Flag indicating that the Cython files need to generate C++ rather than C. + +``SOURCE_FILES`` + The list of Cython source files to be built into Python extension modules. + Note that this function assumes that Cython source files have a one-one + correspondence with extension modules to build, i.e. for every `.pyx` + in SOURCE_FILES we assume that `.pyx` is a Cython source file for which + an extension module `` should be built. + +``LINKED_LIBRARIES`` + The list of libraries that need to be linked into all modules. In RAPIDS, + this list usually contains (at minimum) the corresponding C++ libraries. + +``INSTALL_DIR`` + The path relative to the installation prefix so that it can be converted to + an absolute path in a relocatable way. If not provided, defaults to the path + to CMAKE_CURRENT_SOURCE_DIR relative to PROJECT_SOURCE_DIR. + +#]=======================================================================] +function(rapids_cython_create_modules) + include(${CMAKE_CURRENT_FUNCTION_LIST_DIR}/detail/verify_init.cmake) + rapids_cython_verify_init() + + list(APPEND CMAKE_MESSAGE_CONTEXT "rapids.cython.create_modules") + + set(_rapids_cython_options CXX) + set(_rapids_cython_one_value INSTALL_DIR) + set(_rapids_cython_multi_value SOURCE_FILES LINKED_LIBRARIES) + cmake_parse_arguments(RAPIDS_CYTHON "${_rapids_cython_options}" "${_rapids_cython_one_value}" + "${_rapids_cython_multi_value}" ${ARGN}) + + set(language "C") + if(RAPIDS_CYTHON_CXX) + set(language "CXX") + endif() + + foreach(cython_filename IN LISTS RAPIDS_CYTHON_SOURCE_FILES) + # Generate a reasonable module name. + cmake_path(GET cython_filename FILENAME cython_module) + cmake_path(REMOVE_EXTENSION cython_module) + + add_cython_target(${cython_module} "${cython_filename}" ${language} PY3 OUTPUT_VAR + cythonized_file) + add_library(${cython_module} MODULE ${cythonized_file}) + python_extension_module(${cython_module}) + + # To avoid libraries being prefixed with "lib". + set_target_properties(${cython_module} PROPERTIES PREFIX "") + if(DEFINED RAPIDS_CYTHON_LINKED_LIBRARIES) + target_link_libraries(${cython_module} ${RAPIDS_CYTHON_LINKED_LIBRARIES}) + endif() + + # Compute the install directory relative to the source and rely on installs being relative to + # the CMAKE_PREFIX_PATH for e.g. editable installs. + if(NOT DEFINED RAPIDS_CYTHON_INSTALL_DIR) + cmake_path(RELATIVE_PATH CMAKE_CURRENT_SOURCE_DIR BASE_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE RAPIDS_CYTHON_INSTALL_DIR) + endif() + install(TARGETS ${cython_module} DESTINATION ${RAPIDS_CYTHON_INSTALL_DIR}) + endforeach() +endfunction() diff --git a/rapids-cmake/cython/detail/skbuild_patches.cmake b/rapids-cmake/cython/detail/skbuild_patches.cmake new file mode 100644 index 00000000..b0b69043 --- /dev/null +++ b/rapids-cmake/cython/detail/skbuild_patches.cmake @@ -0,0 +1,54 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +include_guard(GLOBAL) + +#[=======================================================================[.rst: +_set_python_extension_symbol_visibility +--------------------------------------- + +.. versionadded:: v22.06.00 + +The original version of this function in scikit-build runs a linker script to +modify the visibility of symbols. This version is a patch to avoid overwriting +the visibility of symbols because otherwise any library that exports symbols +with external linkage will have the visibility of those symbols changed +undesirably. We can remove this function once this issue is resolved in +scikit-build. + +Issue: https://github.com/scikit-build/scikit-build/issues/668 +PR: https://github.com/scikit-build/scikit-build/pull/703 + +#]=======================================================================] +function(_set_python_extension_symbol_visibility _target) + include(${CMAKE_CURRENT_FUNCTION_LIST_DIR}/verify_init.cmake) + rapids_cython_verify_init() + + if(PYTHON_VERSION_MAJOR VERSION_GREATER 2) + set(_modinit_prefix "PyInit_") + else() + set(_modinit_prefix "init") + endif() + message("_modinit_prefix:${_modinit_prefix}") + if("${CMAKE_C_COMPILER_ID}" STREQUAL "MSVC") + set_target_properties(${_target} PROPERTIES LINK_FLAGS "/EXPORT:${_modinit_prefix}${_target}") + elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(_script_path ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${_target}-version-script.map) + file(WRITE ${_script_path} # Note: The change is to this script, which does not indiscriminately + # mark all non PyInit symbols as local. + "{global: ${_modinit_prefix}${_target}; };") + set_property(TARGET ${_target} APPEND_STRING + PROPERTY LINK_FLAGS " -Wl,--version-script=\"${_script_path}\"") + endif() +endfunction() diff --git a/rapids-cmake/cython/detail/verify_init.cmake b/rapids-cmake/cython/detail/verify_init.cmake new file mode 100644 index 00000000..1aedfcfb --- /dev/null +++ b/rapids-cmake/cython/detail/verify_init.cmake @@ -0,0 +1,34 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +include_guard(GLOBAL) + +#[=======================================================================[.rst: +rapids_cython_verify_init +------------------------- + +.. versionadded:: v22.06.00 + +Simple helper function for rapids-cython components to verify that rapids_cython_init has been called before they proceed. + +.. code-block:: cmake + + rapids_cython_verify_init() + +#]=======================================================================] +function(rapids_cython_verify_init) + if(NOT DEFINED RAPIDS_CYTHON_INITIALIZED) + message(FATAL_ERROR "You must call rapids_cython_init before calling this function") + endif() +endfunction() diff --git a/rapids-cmake/cython/init.cmake b/rapids-cmake/cython/init.cmake new file mode 100644 index 00000000..2dde010c --- /dev/null +++ b/rapids-cmake/cython/init.cmake @@ -0,0 +1,62 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +include_guard(GLOBAL) + +#[=======================================================================[.rst: +rapids_cython_init +------------------ + +.. versionadded:: v22.06.00 + +Perform standard initialization of any CMake build using scikit-build to create Python extension modules with Cython. + +.. code-block:: cmake + + rapids_cython_init() + +Result Variables +^^^^^^^^^^^^^^^^ + :cmake:variable:`RAPIDS_CYTHON_INITIALIZED` will be set to TRUE. + :cmake:variable:`CYTHON_FLAGS` will be set to a standard set of a flags to pass to the command line cython invocation. + +#]=======================================================================] +macro(rapids_cython_init) + list(APPEND CMAKE_MESSAGE_CONTEXT "rapids.cython.init") + # Only initialize once. + if(NOT DEFINED RAPIDS_CYTHON_INITIALIZED) + # Verify that we are using scikit-build. + if(NOT DEFINED SKBUILD) + message(WARNING "rapids-cython expects scikit-build to be active before being used. \ + The SKBUILD variable is not currently set, indicating that scikit-build \ + is not active, so builds may behave unexpectedly.") + else() + # Access the variable to avoid unused variable warnings." + message(TRACE "Accessing SKBUILD variable ${SKBUILD}") + endif() + + find_package(PythonExtensions REQUIRED) + find_package(Cython REQUIRED) + + # Incorporate scikit-build patches. + include("${rapids-cmake-dir}/cython/detail/skbuild_patches.cmake") + + if(NOT CYTHON_FLAGS) + set(CYTHON_FLAGS "--directive binding=True,embedsignature=True,always_allow_keywords=True") + endif() + + # Flag + set(RAPIDS_CYTHON_INITIALIZED TRUE) + endif() +endmacro() diff --git a/rapids-cmake/rapids-cython.cmake b/rapids-cmake/rapids-cython.cmake new file mode 100644 index 00000000..ff5eef34 --- /dev/null +++ b/rapids-cmake/rapids-cython.cmake @@ -0,0 +1,19 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include_guard(GLOBAL) + +include(${CMAKE_CURRENT_LIST_DIR}/cython/init.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cython/create_modules.cmake) diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 523e1d1a..60173988 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -35,6 +35,7 @@ if(RAPIDS_CMAKE_ENABLE_DOWNLOAD_TESTS) setup_cpm_cache() add_subdirectory(cpm) + add_subdirectory(cython) add_subdirectory(other) endif() diff --git a/testing/cython/CMakeLists.txt b/testing/cython/CMakeLists.txt new file mode 100644 index 00000000..38f7f965 --- /dev/null +++ b/testing/cython/CMakeLists.txt @@ -0,0 +1,23 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +add_cmake_config_test(rapids-cython.cmake) + +add_cmake_config_test(init.cmake) +add_cmake_config_test(create_modules_errors.cmake SHOULD_FAIL "You must call rapids_cython_init before calling this function") + +add_cmake_config_test(create_modules) +add_cmake_config_test(create_modules_with_library) diff --git a/testing/cython/create_modules/CMakeLists.txt b/testing/cython/create_modules/CMakeLists.txt new file mode 100644 index 00000000..1ae3a334 --- /dev/null +++ b/testing/cython/create_modules/CMakeLists.txt @@ -0,0 +1,40 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.20) + +include(${rapids-cmake-dir}/cython/create_modules.cmake) +include(${rapids-cmake-dir}/cython/init.cmake) + +project(rapids_cython-create_modules LANGUAGES C CXX) + +# Silence warning about running without scikit-build. +set(SKBUILD ON) + +# Ensure that scikit-build's CMake files are discoverable. The glob is to +# capture the current git commit hash. +file(GLOB skbuild_resource_dir LIST_DIRECTORIES ON "${CPM_SOURCE_CACHE}/skbuild/*/skbuild/resources/cmake") +LIST(APPEND CMAKE_MODULE_PATH "${skbuild_resource_dir}") + +rapids_cython_init() + +# Test that an empty invocation works. +rapids_cython_create_modules() + +# Test that a basic invocation with a single file works. +rapids_cython_create_modules( + SOURCE_FILES test.pyx + ) diff --git a/testing/cython/create_modules/test.pyx b/testing/cython/create_modules/test.pyx new file mode 100644 index 00000000..e69de29b diff --git a/testing/cython/create_modules_errors.cmake b/testing/cython/create_modules_errors.cmake new file mode 100644 index 00000000..0a4be205 --- /dev/null +++ b/testing/cython/create_modules_errors.cmake @@ -0,0 +1,21 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include(${rapids-cmake-dir}/cython/create_modules.cmake) + +# Test that a invocation without calling rapids_cython_init fails. +rapids_cython_create_modules( + SOURCE_FILES test.pyx + ) diff --git a/testing/cython/create_modules_with_library/CMakeLists.txt b/testing/cython/create_modules_with_library/CMakeLists.txt new file mode 100644 index 00000000..e6727f0c --- /dev/null +++ b/testing/cython/create_modules_with_library/CMakeLists.txt @@ -0,0 +1,39 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +cmake_minimum_required(VERSION 3.20) + +include(${rapids-cmake-dir}/cython/create_modules.cmake) +include(${rapids-cmake-dir}/cython/init.cmake) + +project(rapids_cython-create_modules LANGUAGES C CXX) + +# Silence warning about running without scikit-build. +set(SKBUILD ON) + +# Ensure that scikit-build's CMake files are discoverable. The glob is to +# capture the current git commit hash. +file(GLOB skbuild_resource_dir LIST_DIRECTORIES ON "${CPM_SOURCE_CACHE}/skbuild/*/skbuild/resources/cmake") +LIST(APPEND CMAKE_MODULE_PATH "${skbuild_resource_dir}") + +rapids_cython_init() + +# Test that we can specify a (fake) library. +add_library(rapids_cython_test_library test) +rapids_cython_create_modules( + SOURCE_FILES test.pyx + LINKED_LIBRARIES rapids_cython_test_library + ) diff --git a/testing/cython/create_modules_with_library/test.pyx b/testing/cython/create_modules_with_library/test.pyx new file mode 100644 index 00000000..e69de29b diff --git a/testing/cython/init.cmake b/testing/cython/init.cmake new file mode 100644 index 00000000..448f4966 --- /dev/null +++ b/testing/cython/init.cmake @@ -0,0 +1,53 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include(${rapids-cmake-dir}/cython/init.cmake) + +# Silence warning about running without scikit-build. +set(SKBUILD ON) + +# Ensure that scikit-build's CMake files are discoverable. The glob is to +# capture the current git commit hash. +file(GLOB skbuild_resource_dir LIST_DIRECTORIES ON "${CPM_SOURCE_CACHE}/skbuild/*/skbuild/resources/cmake") +LIST(APPEND CMAKE_MODULE_PATH "${skbuild_resource_dir}") + +# Test that rapids_cython_init initializes the expected variables. +rapids_cython_init() +if(NOT DEFINED RAPIDS_CYTHON_INITIALIZED) + message(FATAL_ERROR "rapids_cython_init didn't correctly set RAPIDS_CYTHON_INITIALIZED") +endif() + +string(REGEX MATCHALL ".*--directive.*" matches "${CYTHON_FLAGS}") +list(LENGTH matches num_directives) + +if(NOT CYTHON_FLAGS OR NOT num_directives EQUAL 1) + message(FATAL_ERROR "rapids_cython_init didn't correctly set CYTHON_FLAGS") +endif() + +if(NOT COMMAND _set_python_extension_symbol_visibility) + message(FATAL_ERROR "rapids_cython_init didn't create the _set_python_extension_symbol_visibility command") +endif() + +# Test that rapids_cython_init is idempotent. +rapids_cython_init() +string(REGEX MATCHALL ".*--directive.*" matches "${CYTHON_FLAGS}") +list(LENGTH matches num_directives) + +if(NOT num_directives EQUAL 1) + message(FATAL_ERROR "rapids_cython_init is not idempotent, num_directives = ${num_directives}") +endif() + +# Unset the cached CYTHON_FLAGS variable for future runs of the test to behave as expected. +unset(CYTHON_FLAGS CACHE) diff --git a/testing/cython/rapids-cython.cmake b/testing/cython/rapids-cython.cmake new file mode 100644 index 00000000..3825ebb7 --- /dev/null +++ b/testing/cython/rapids-cython.cmake @@ -0,0 +1,16 @@ +#============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include(${rapids-cmake-dir}/rapids-cython.cmake) diff --git a/testing/utils/fill_cache/CMakeLists.txt b/testing/utils/fill_cache/CMakeLists.txt index a451bb38..249476dd 100644 --- a/testing/utils/fill_cache/CMakeLists.txt +++ b/testing/utils/fill_cache/CMakeLists.txt @@ -36,6 +36,7 @@ rapids_cpm_nvbench(DOWNLOAD_ONLY ON) rapids_cpm_rmm(DOWNLOAD_ONLY ON) rapids_cpm_spdlog(DOWNLOAD_ONLY ON) rapids_cpm_thrust(temp DOWNLOAD_ONLY ON) - - - +rapids_cpm_find(skbuild 0.14.1 + GIT_REPOSITORY https://github.com/scikit-build/scikit-build.git + GIT_TAG 0.14.1 + )