Skip to content

Commit

Permalink
Add ability to register custom ops by specifying a function name (#14177
Browse files Browse the repository at this point in the history 
)

### Description
<!-- Describe your changes. -->
Use dlsym/GetProcAddress to lookup a custom ops registration function by
name and call it.

This will be better on mobile platforms where the custom ops library is
linked against, and there isn't necessarily a filesystem that a library
path can be loaded from.

Alternative is to wire up passing in the address of the function, but
that has multiple complications which differ by platform.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
Enable using ort and ort-ext packages on mobile platforms.

Co-authored-by: Edward Chen <[email protected]>
  • Loading branch information
@skottmckay @edgchen1
skottmckay and edgchen1 authored Jan 12, 2023
1 parent f5b4b0f commit b9ecd42
Show file tree
Hide file tree
Showing 19 changed files with 2,401 additions and 2,145 deletions.
22 changes: 22 additions & 0 deletions ThirdPartyNotices.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5144,3 +5144,25 @@ google/sentencepiece, https://github.com/google/sentencepiece
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

_____

dlfcn-win32/dlfcn-win32 is licensed under the MIT License

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
42 changes: 30 additions & 12 deletions cgmanifests/cgmanifest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,27 @@
}
},
{
"component": {
"Type": "maven",
"maven": { "GroupId": "org.junit.platform", "ArtifactId": "junit-platform-console-standalone", "Version": "1.6.2" },
"DevelopmentDependency": true
}
"component": {
"Type": "maven",
"maven": {
"GroupId": "org.junit.platform",
"ArtifactId": "junit-platform-console-standalone",
"Version": "1.6.2"
},
"DevelopmentDependency": true
}
},
{
"component": {
"Type": "maven",
"maven": { "GroupId": "com.google.protobuf", "ArtifactId": "protobuf-java", "Version": "3.21.7" },
"DevelopmentDependency": true
}
},
"component": {
"Type": "maven",
"maven": {
"GroupId": "com.google.protobuf",
"ArtifactId": "protobuf-java",
"Version": "3.21.7"
},
"DevelopmentDependency": true
}
},
{
"component": {
"type": "git",
Expand Down Expand Up @@ -535,7 +543,17 @@
"DownloadUrl": "http://security.ubuntu.com/ubuntu/pool/main/s/sqlite3/libsqlite3-dev_3.22.0-1ubuntu0.4_amd64.deb"
}
}
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "9d0ef119d9fcb9139f831adc224857b791c81140",
"repositoryUrl": "https://github.com/dlfcn-win32/dlfcn-win32.git"
},
"comments": "dlfcn-win32"
}
}
],
"Version": 1
}
140 changes: 86 additions & 54 deletions cmake/onnxruntime_unittests.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@ function(AddTest)
if (_UT_DEPENDS)
add_dependencies(${_UT_TARGET} ${_UT_DEPENDS})
endif(_UT_DEPENDS)

if(_UT_DYN)
target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} GTest::gtest GTest::gmock onnxruntime ${CMAKE_DL_LIBS}
Threads::Threads)
target_compile_definitions(${_UT_TARGET} PRIVATE -DUSE_ONNXRUNTIME_DLL)
else()
target_link_libraries(${_UT_TARGET} PRIVATE ${_UT_LIBS} GTest::gtest GTest::gmock ${onnxruntime_EXTERNAL_LIBRARIES})
endif()

onnxruntime_add_include_to_target(${_UT_TARGET} date_interface flatbuffers::flatbuffers)
target_include_directories(${_UT_TARGET} PRIVATE ${TEST_INC_DIR})
if (onnxruntime_USE_CUDA)
Expand All @@ -61,10 +63,12 @@ function(AddTest)
target_include_directories(${_UT_TARGET} PRIVATE ${NCCL_INCLUDE_DIRS})
endif()
endif()

if(MSVC)
target_compile_options(${_UT_TARGET} PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>"
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>")
endif()

if (WIN32)
# include dbghelp in case tests throw an ORT exception, as that exception includes a stacktrace, which requires dbghelp.
target_link_libraries(${_UT_TARGET} PRIVATE debug dbghelp)
Expand Down Expand Up @@ -382,6 +386,7 @@ endif()
set (ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR "${TEST_SRC_DIR}/shared_lib")
set (ONNXRUNTIME_GLOBAL_THREAD_POOLS_TEST_SRC_DIR "${TEST_SRC_DIR}/global_thread_pools")
set (ONNXRUNTIME_API_TESTS_WITHOUT_ENV_SRC_DIR "${TEST_SRC_DIR}/api_tests_without_env")
set (ONNXRUNTIME_CUSTOM_OP_REGISTRATION_TEST_SRC_DIR "${TEST_SRC_DIR}/custom_op_registration")

set (onnxruntime_shared_lib_test_SRC
${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/test_fixture.h
Expand Down Expand Up @@ -1163,24 +1168,27 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
if (CMAKE_SYSTEM_NAME STREQUAL "Android")
list(APPEND onnxruntime_shared_lib_test_LIBS ${android_shared_libs})
endif()

AddTest(DYN
TARGET onnxruntime_shared_lib_test
SOURCES ${onnxruntime_shared_lib_test_SRC} ${onnxruntime_unittest_main_src}
LIBS ${onnxruntime_shared_lib_test_LIBS}
DEPENDS ${all_dependencies}
)

if (CMAKE_SYSTEM_NAME STREQUAL "Android")
target_sources(onnxruntime_shared_lib_test PRIVATE
"${ONNXRUNTIME_ROOT}/core/platform/android/cxa_demangle.cc"
"${TEST_SRC_DIR}/platform/android/cxa_demangle_test.cc"
)
target_compile_definitions(onnxruntime_shared_lib_test PRIVATE USE_DUMMY_EXA_DEMANGLE=1)
endif()

if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
add_custom_command(
TARGET onnxruntime_shared_lib_test POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
${TEST_DATA_DES}
${TEST_DATA_SRC}
$<TARGET_FILE_DIR:onnxruntime_shared_lib_test>/testdata)
endif()

Expand Down Expand Up @@ -1343,70 +1351,94 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
endif()

if (NOT onnxruntime_BUILD_WEBASSEMBLY)
if (onnxruntime_USE_CUDA)
onnxruntime_add_shared_library_module(custom_op_library ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/cuda_ops.cu
${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc)
target_include_directories(custom_op_library PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
if (HAS_QSPECTRE)
target_compile_options(custom_op_library PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:-Xcompiler /Qspectre>")
if (onnxruntime_USE_CUDA)
onnxruntime_add_shared_library(custom_op_library ${ONNXRUNTIME_SHARED_LIB_TEST_SRC_DIR}/cuda_ops.cu
${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc)
target_include_directories(custom_op_library PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
if (HAS_QSPECTRE)
target_compile_options(custom_op_library PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:-Xcompiler /Qspectre>")
endif()
else()
onnxruntime_add_shared_library(custom_op_library ${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc)
endif()
else()
onnxruntime_add_shared_library_module(custom_op_library ${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.cc)
endif()
target_include_directories(custom_op_library PRIVATE ${REPO_ROOT}/include)
target_link_libraries(custom_op_library PRIVATE ${GSL_TARGET})
if(UNIX)
if (APPLE)
set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker -dead_strip")

target_include_directories(custom_op_library PRIVATE ${REPO_ROOT}/include)
target_link_libraries(custom_op_library PRIVATE ${GSL_TARGET})

if(UNIX)
if (APPLE)
set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker -dead_strip")
else()
set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker --version-script=${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.lds -Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack")
endif()
else()
set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-Xlinker --version-script=${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.lds -Xlinker --no-undefined -Xlinker --gc-sections -z noexecstack")
set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-DEF:${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.def")
if (NOT onnxruntime_USE_CUDA)
target_compile_options(custom_op_library PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler /wd26409>"
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/wd26409>")
endif()
endif()
else()
set(ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG "-DEF:${TEST_SRC_DIR}/testdata/custom_op_library/custom_op_library.def")
if (NOT onnxruntime_USE_CUDA)
target_compile_options(custom_op_library PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler /wd26409>"
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/wd26409>")
set_property(TARGET custom_op_library APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG})

if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
if (onnxruntime_BUILD_JAVA AND NOT onnxruntime_ENABLE_STATIC_ANALYSIS)
message(STATUS "Running Java tests")
# native-test is added to resources so custom_op_lib can be loaded
# and we want to symlink it there
set(JAVA_NATIVE_TEST_DIR ${JAVA_OUTPUT_DIR}/native-test)
file(MAKE_DIRECTORY ${JAVA_NATIVE_TEST_DIR})

# delegate to gradle's test runner
if(WIN32)
add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:custom_op_library>
${JAVA_NATIVE_TEST_DIR}/$<TARGET_FILE_NAME:custom_op_library>)
# On windows ctest requires a test to be an .exe(.com) file
# So there are two options 1) Install Chocolatey and its gradle package
# That package would install gradle.exe shim to its bin so ctest could run gradle.exe
# 2) With standard installation we get gradle.bat. We delegate execution to a separate .cmake file
# That can handle both .exe and .bat
add_test(NAME onnxruntime4j_test COMMAND ${CMAKE_COMMAND}
-DGRADLE_EXECUTABLE=${GRADLE_EXECUTABLE}
-DBIN_DIR=${CMAKE_CURRENT_BINARY_DIR}
-DREPO_ROOT=${REPO_ROOT}
${ORT_PROVIDER_FLAGS}
-P ${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime_java_unittests.cmake)
else()
add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:custom_op_library>
${JAVA_NATIVE_TEST_DIR}/$<TARGET_LINKER_FILE_NAME:custom_op_library>)
add_test(NAME onnxruntime4j_test COMMAND ${GRADLE_EXECUTABLE} cmakeCheck -DcmakeBuildDir=${CMAKE_CURRENT_BINARY_DIR} ${ORT_PROVIDER_FLAGS}
WORKING_DIRECTORY ${REPO_ROOT}/java)
endif()
set_property(TEST onnxruntime4j_test APPEND PROPERTY DEPENDS onnxruntime4j_jni)
endif()
endif()
endif()
set_property(TARGET custom_op_library APPEND_STRING PROPERTY LINK_FLAGS ${ONNXRUNTIME_CUSTOM_OP_LIB_LINK_FLAG})

if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
if (onnxruntime_BUILD_JAVA AND NOT onnxruntime_ENABLE_STATIC_ANALYSIS)
message(STATUS "Running Java tests")
# native-test is added to resources so custom_op_lib can be loaded
# and we want to symlink it there
set(JAVA_NATIVE_TEST_DIR ${JAVA_OUTPUT_DIR}/native-test)
file(MAKE_DIRECTORY ${JAVA_NATIVE_TEST_DIR})

# delegate to gradle's test runner
if(WIN32)
add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:custom_op_library>
${JAVA_NATIVE_TEST_DIR}/$<TARGET_FILE_NAME:custom_op_library>)
# On windows ctest requires a test to be an .exe(.com) file
# So there are two options 1) Install Chocolatey and its gradle package
# That package would install gradle.exe shim to its bin so ctest could run gradle.exe
# 2) With standard installation we get gradle.bat. We delegate execution to a separate .cmake file
# That can handle both .exe and .bat
add_test(NAME onnxruntime4j_test COMMAND ${CMAKE_COMMAND}
-DGRADLE_EXECUTABLE=${GRADLE_EXECUTABLE}
-DBIN_DIR=${CMAKE_CURRENT_BINARY_DIR}
-DREPO_ROOT=${REPO_ROOT}
${ORT_PROVIDER_FLAGS}
-P ${CMAKE_CURRENT_SOURCE_DIR}/onnxruntime_java_unittests.cmake)
else()
add_custom_command(TARGET custom_op_library POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:custom_op_library>
${JAVA_NATIVE_TEST_DIR}/$<TARGET_LINKER_FILE_NAME:custom_op_library>)
add_test(NAME onnxruntime4j_test COMMAND ${GRADLE_EXECUTABLE} cmakeCheck -DcmakeBuildDir=${CMAKE_CURRENT_BINARY_DIR} ${ORT_PROVIDER_FLAGS}
WORKING_DIRECTORY ${REPO_ROOT}/java)
endif()
set_property(TEST onnxruntime4j_test APPEND PROPERTY DEPENDS onnxruntime4j_jni)
if (onnxruntime_BUILD_SHARED_LIB AND (NOT onnxruntime_MINIMAL_BUILD OR onnxruntime_MINIMAL_BUILD_CUSTOM_OPS))
set (onnxruntime_customopregistration_test_SRC
${ONNXRUNTIME_CUSTOM_OP_REGISTRATION_TEST_SRC_DIR}/test_registercustomops.cc)

set(onnxruntime_customopregistration_test_LIBS custom_op_library onnxruntime_common onnxruntime_test_utils)
AddTest(DYN
TARGET onnxruntime_customopregistration_test
SOURCES ${onnxruntime_customopregistration_test_SRC} ${onnxruntime_unittest_main_src}
LIBS ${onnxruntime_customopregistration_test_LIBS}
DEPENDS ${all_dependencies}
)

if (CMAKE_SYSTEM_NAME STREQUAL "iOS")
add_custom_command(
TARGET onnxruntime_customopregistration_test POST_BUILD
COMMAND ${CMAKE_COMMAND} -E copy_directory
${TEST_DATA_SRC}
$<TARGET_FILE_DIR:onnxruntime_customopregistration_test>/testdata)
endif()
endif()
endif()
endif()

# Build custom op library that returns an error OrtStatus when the exported RegisterCustomOps function is called.
if (NOT onnxruntime_BUILD_WEBASSEMBLY AND (NOT onnxruntime_MINIMAL_BUILD OR onnxruntime_MINIMAL_BUILD_CUSTOM_OPS))
onnxruntime_add_shared_library_module(custom_op_invalid_library
${TEST_SRC_DIR}/testdata/custom_op_invalid_library/custom_op_library.h
${TEST_SRC_DIR}/testdata/custom_op_invalid_library/custom_op_library.cc)
target_include_directories(custom_op_invalid_library PRIVATE ${REPO_ROOT}/include/onnxruntime/core/session)

Expand Down
Loading

0 comments on commit b9ecd42

Please sign in to comment.