Improved cross-compilation with python

.github/workflows/linux_riscv.yml

@@ -110,6 +110,7 @@ jobs:
             git submodule update --init -- ${OPENVINO_REPO}/thirdparty/telemetry
             git submodule update --init -- ${OPENVINO_REPO}/src/plugins/intel_cpu
             git submodule update --init -- ${OPENVINO_REPO}/thirdparty/open_model_zoo
+            git submodule update --init -- ${OPENVINO_REPO}/thirdparty/flatbuffers/flatbuffers
           popd
 
       #
@@ -192,7 +193,6 @@ jobs:
             -DENABLE_INTEL_GPU=ON \
             -DENABLE_PYTHON=ON \
             -DENABLE_WHEEL=ON \
-            -DPYTHON_MODULE_EXTENSION=$(riscv64-linux-gnu-python3-config --extension-suffix) \
             -DENABLE_TESTS=ON \
             -DENABLE_PYTHON_PACKAGING=ON \
             -DENABLE_SYSTEM_PROTOBUF=ON \

CMakeLists.txt

@@ -51,6 +51,7 @@ endif()
 
 # resolving dependencies for the project
 message (STATUS "CMAKE_VERSION ......................... " ${CMAKE_VERSION})
+message (STATUS "CMAKE_CROSSCOMPILING .................. " ${CMAKE_CROSSCOMPILING})
 message (STATUS "OpenVINO_SOURCE_DIR ................... " ${OpenVINO_SOURCE_DIR})
 message (STATUS "OpenVINO_BINARY_DIR ................... " ${OpenVINO_BINARY_DIR})
 message (STATUS "CMAKE_GENERATOR ....................... " ${CMAKE_GENERATOR})

cmake/developer_package/OpenVINODeveloperScriptsConfig.cmake

@@ -92,6 +92,7 @@ endfunction()
 #
 
 include(cross_compile/find_commands)
+include(cross_compile/python_helpers)
 include(cross_compile/native_compile)
 
 #

cmake/developer_package/cross_compile/find_commands.cmake

@@ -2,6 +2,31 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
+# The two functions below are used to allow cmake find search for host system
+# locations during find_package
+
+macro(ov_cross_compile_define_debian_arch)
+    if(CMAKE_HOST_LINUX AND CMAKE_CROSSCOMPILING)
+        set(_old_CMAKE_LIBRARY_ARCHITECTURE ${CMAKE_LIBRARY_ARCHITECTURE})
+        # without this WA cmake does not search in <triplet> subfolder
+        # see https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure
+        if(HOST_X86_64)
+            set(CMAKE_LIBRARY_ARCHITECTURE "x86_64-linux-gnu")
+        elseif(HOST_AARCH64)
+            set(CMAKE_LIBRARY_ARCHITECTURE "aarch64-linux-gnu")
+        elseif(HOST_RISCV64)
+            set(CMAKE_LIBRARY_ARCHITECTURE "riscv64-linux-gnu")
+        endif()
+    endif()
+endmacro()
+
+macro(ov_cross_compile_define_debian_arch_reset)
+    if(CMAKE_HOST_LINUX AND CMAKE_CROSSCOMPILING)
+        set(CMAKE_LIBRARY_ARCHITECTURE ${_old_CMAKE_LIBRARY_ARCHITECTURE})
+        unset(_old_CMAKE_LIBRARY_ARCHITECTURE)
+    endif()
+endmacro()
+
 # Search packages for the host system instead of packages for the target system
 # in case of cross compilation these macros should be defined by the toolchain file
 

cmake/developer_package/cross_compile/python_helpers.cmake

@@ -0,0 +1,74 @@
+# Copyright (C) 2018-2024  Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+function(ov_detect_python_module_extension)
+    if(NOT ENABLE_PYTHON)
+        # python is just disabled
+        return()
+    endif()
+
+    if(PYTHON_MODULE_EXTENSION)
+        # exit if it's already defined
+        return()
+    endif()
+
+    if(NOT CMAKE_CROSSCOMPILING)
+        # in case of native compilation FindPython3.cmake properly detects PYTHON_MODULE_EXTENSION
+        return()
+    endif()
+
+    if(RISCV64)
+        set(python3_config riscv64-linux-gnu-python3-config)
+    elseif(AARCH64)
+        set(python3_config aarch64-linux-gnu-python3-config)
+    elseif(X86_64)
+        set(python3_config x86_64-linux-gnu-python3-config)
+    else()
+        message(WARNING "Python cross-compilation warning: ${OV_ARCH} is unknown for python build. Please, specify PYTHON_MODULE_EXTENSION explicitly")
+    endif()
+
+    find_host_program(python3_config_exec NAMES ${python3_config})
+    if(python3_config_exec)
+        execute_process(COMMAND ${python3_config_exec} --extension-suffix
+                        RESULT_VARIABLE EXIT_CODE
+                        OUTPUT_VARIABLE PYTHON_MODULE_EXTENSION
+                        ERROR_VARIABLE ERROR_TEXT
+                        OUTPUT_STRIP_TRAILING_WHITESPACE)
+        if(NOT EXIT_CODE EQUAL 0)
+            message(FATAL_ERROR "Internal error: failed to execute ${python3_config_exec}")
+        endif()
+        set(PYTHON_MODULE_EXTENSION ${PYTHON_MODULE_EXTENSION} PARENT_SCOPE)
+    else()
+        message(FATAL_ERROR [=[PYTHON_MODULE_EXTENSION will not be properly detected. Please, either:
+    1. Install libpython3-dev for target architecture
+    2. Explicitly specify PYTHON_MODULE_EXTENSION
+    ]=])
+    endif()
+endfunction()
+
+# Wrapper for find_package(Python3) to allow cross-compilation
+macro(ov_find_python3 find_package_mode)
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
+        set(python3_development_component Development.Module)
+    else()
+        set(python3_development_component Development)
+    endif()
+
+    if(CMAKE_CROSSCOMPILING AND LINUX)
+        # allow to find python headers from host in case of cross-compilation
+        # e.g. install libpython3-dev:<foreign arch> and finds its headers
+        set(_old_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${CMAKE_FIND_ROOT_PATH_MODE_INCLUDE})
+        set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)
+        ov_cross_compile_define_debian_arch()
+    endif()
+
+    find_package(Python3 ${find_package_mode} COMPONENTS Interpreter ${python3_development_component})
+
+    if(CMAKE_CROSSCOMPILING AND LINUX)
+        ov_cross_compile_define_debian_arch_reset()
+        set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ${_old_CMAKE_FIND_ROOT_PATH_MODE_INCLUDE})
+    endif()
+
+    unset(python3_development_component)
+endmacro()

cmake/features.cmake

@@ -151,8 +151,8 @@ else()
     set(ENABLE_SYSTEM_LIBS_DEFAULT OFF)
 endif()
 
-if(ANDROID)
-    # when protobuf from /usr/include is used, then Android toolchain ignores include paths
+if(CMAKE_CROSSCOMPILING AND (ANDROID OR RISCV64))
+    # when protobuf from /usr/include is used, then Android / Risc-V toolchain ignores include paths
     # but if we build for Android using vcpkg / conan / etc where flatbuffers is not located in
     # the /usr/include folders, we can still use 'system' flatbuffers
     set(ENABLE_SYSTEM_FLATBUFFERS_DEFAULT OFF)

src/bindings/python/CMakeLists.txt

@@ -71,7 +71,8 @@ function(ov_check_python_build_conditions)
         set(message_mode WARNING)
     endif()
 
-    find_package(Python3 ${find_package_mode} COMPONENTS Interpreter ${python3_development_component})
+    ov_find_python3(${find_package_mode})
+
     if(Python3_Development.Module_FOUND OR Python3_Development_FOUND)
         message(STATUS "Python3 executable: ${Python3_EXECUTABLE}")
         message(STATUS "Python3 version: ${Python3_VERSION}")
@@ -82,7 +83,9 @@ function(ov_check_python_build_conditions)
         if(Python3_SOABI)
             message(STATUS "Python3 SOABI: ${Python3_SOABI}")
         endif()
+        ov_detect_python_module_extension()
         if(PYTHON_MODULE_EXTENSION)
+            set(PYTHON_MODULE_EXTENSION ${PYTHON_MODULE_EXTENSION} PARENT_SCOPE)
             message(STATUS "PYTHON_MODULE_EXTENSION: ${PYTHON_MODULE_EXTENSION}")
         endif()
         message(STATUS "Python3 include dirs: ${Python3_INCLUDE_DIRS}")
@@ -103,7 +106,6 @@ function(ov_check_python_build_conditions)
     else()
         set(ENABLE_PYTHON_DEFAULT OFF PARENT_SCOPE)
     endif()
-
 endfunction()
 
 ov_check_python_build_conditions()
@@ -207,7 +209,7 @@ endif()
 # search for FindPython3.cmake instead of legacy modules
 set(PYBIND11_FINDPYTHON ON)
 
-find_package(Python3 REQUIRED COMPONENTS Interpreter ${python3_development_component})
+ov_find_python3(REQUIRED)
 find_package(pybind11 ${pybind11_min_version} QUIET)
 
 if(NOT pybind11_FOUND)

thirdparty/dependencies.cmake

@@ -381,25 +381,13 @@ endif()
 
 if(ENABLE_OV_TF_LITE_FRONTEND)
     if(ENABLE_SYSTEM_FLATBUFFERS)
-        if(CMAKE_HOST_LINUX)
-            set(_old_flat_CMAKE_LIBRARY_ARCHITECTURE ${CMAKE_LIBRARY_ARCHITECTURE})
-            # without this WA cmake does not search in <triplet> subfolder
-            # see https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure
-            if(HOST_X86_64)
-                set(CMAKE_LIBRARY_ARCHITECTURE "x86_64-linux-gnu")
-            elseif(HOST_AARCH64)
-                set(CMAKE_LIBRARY_ARCHITECTURE "aarch64-linux-gnu")
-            endif()
-        endif()
+        ov_cross_compile_define_debian_arch()
 
         # on new Ubuntu versions like 23.04 we have config called FlatBuffersConfig.cmake
         # so, we need to provide alternative names
         find_host_package(Flatbuffers QUIET NAMES Flatbuffers FlatBuffers NO_CMAKE_FIND_ROOT_PATH)
 
-        if(DEFINED _old_flat_CMAKE_LIBRARY_ARCHITECTURE)
-            set(CMAKE_LIBRARY_ARCHITECTURE ${_old_flat_CMAKE_LIBRARY_ARCHITECTURE})
-            unset(_old_flat_CMAKE_LIBRARY_ARCHITECTURE)
-        endif()
+        ov_cross_compile_define_debian_arch_reset()
     endif()
 
     if(Flatbuffers_FOUND)

tools/mo/unit_tests/mock_mo_frontend/mock_mo_python_api/CMakeLists.txt

@@ -36,21 +36,17 @@ endif()
 # search for FindPython3.cmake instead of legacy modules
 set(PYBIND11_FINDPYTHON ON)
 
+ov_detect_python_module_extension()
+
 find_package(pybind11 ${pybind11_min_version} QUIET)
 
 if(NOT pybind11_FOUND)
     add_subdirectory(${OpenVINO_SOURCE_DIR}/src/bindings/python/thirdparty/pybind11
-                    ${CMAKE_CURRENT_BINARY_DIR}/pybind11_build
-                    EXCLUDE_FROM_ALL)
-endif()
-
-if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-    set(python3_development_component Development.Module)
-else()
-    set(python3_development_component Development)
+                     ${CMAKE_CURRENT_BINARY_DIR}/pybind11_build
+                     EXCLUDE_FROM_ALL)
 endif()
 
-find_package(Python3 REQUIRED COMPONENTS Interpreter ${python3_development_component})
+ov_find_python3(REQUIRED)
 pybind11_add_module(${PYBIND_FE_NAME} MODULE NO_EXTRAS ${PYBIND_FE_SRC})
 
 target_link_libraries(${PYBIND_FE_NAME} PRIVATE openvino::runtime)