microsoft · ke1337 · Sep 2, 2019 · May 25, 2019 · May 28, 2019 · May 28, 2019
diff --git a/BUILD.md b/BUILD.md
@@ -86,6 +86,7 @@ The complete list of build options can be found by running `./build.sh (or ./bui
 * [Intel nGraph](#nGraph)
 * [Intel OpenVINO](#openvino)
 * [Android NNAPI](#Android)
+* [Nuphar](#Nuphar)
 
 **Options**
 * [OpenMP](#OpenMP)
@@ -270,6 +271,46 @@ Note: For 32-bit devices, replace `-DANDROID_ABI=arm64-v8a` to `-DANDROID_ABI=ar
 
 ---
 
+### Nuphar
+ONNX Runtime supports Nuphar execution provider (released as preview). It is an execution provider built on top of [TVM](https://github.com/dmlc/tvm) and [LLVM](https://llvm.org). Currently it targets to X64 CPU.
+
+The Nuphar execution provider for ONNX Runtime is built and tested with LLVM 6.0.1. Because of TVM's requirement when building with LLVM, you need to build LLVM from source:
+
+Window with Visual Studio 2017: (Note here builds release flavor. Debug build of LLVM would be needed to build with Debug flavor of ONNX Runtime)
+```
+REM download llvm source code 6.0.1 and unzip to \llvm\source\path, then install to \llvm\install\path
+cd \llvm\source\path
+mkdir build
+cd build
+cmake .. -G "Visual Studio 15 2017 Win64" -DLLVM_TARGETS_TO_BUILD=X86
+msbuild llvm.sln /maxcpucount /p:Configuration=Release /p:Platform=x64
+cmake -DCMAKE_INSTALL_PREFIX=\llvm\install\path -DBUILD_TYPE=Release -P cmake_install.cmake
+```
+
+Linux:
+```
+# download llvm source code 6.0.1 and unzip to /llvm/source/path, then install to /llvm/install/path
+cd /llvm/source/path
+mkdir build
+cd build
+cmake .. -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release
+cmake --build.
+cmake -DCMAKE_INSTALL_PREFIX=/llvm/install/path -DBUILD_TYPE=Release -P cmake_install.cmake
+```
+
+Then you can build from source by using following command from the onnxruntime directory:
+Windows:
+```
+build.bat --use_tvm --use_llvm --llvm_path=\llvm\install\path\lib\cmake\llvm --use_mklml --use_nuphar --build_shared_lib --build_csharp --enable_pybind --config=Release
+```
+
+Linux:
+```
+./build.sh --use_tvm --use_llvm --llvm_path=/llvm/install/path/lib/cmake/llvm --use_mklml --use_nuphar --build_shared_lib --build_csharp --enable_pybind --config=Release
+```
+
+---
+
 ## Options
 ### OpenMP
 ```

diff --git a/README.md b/README.md
@@ -57,6 +57,7 @@ Currently ONNX Runtime supports the following accelerators:
 * CUDA
 * [TensorRT](https://github.com/microsoft/onnxruntime/blob/master/docs/execution_providers/TensorRT-ExecutionProvider.md)
 * [OpenVINO](https://github.com/microsoft/onnxruntime/blob/master/docs/execution_providers/OpenVINO-ExecutionProvider.md)
+* [Nuphar](docs/execution_providers/Nuphar-ExecutionProvider.md)
 
 Not all variations are supported in the [official release builds](#apis-and-official-builds), but can be built from source following [these instructions](https://github.com/Microsoft/onnxruntime/blob/master/BUILD.md). Find Dockerfiles [here](https://github.com/microsoft/onnxruntime/tree/master/dockerfiles).
 

diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -350,8 +350,13 @@ if (onnxruntime_USE_TVM)
   add_definitions(-DUSE_TVM)
 
   set(onnxruntime_tvm_libs onnxruntime_codegen_tvm)
-  list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm nnvm_compiler)
-
+  # needs to link with stdc++fs in Linux
+  if(UNIX)
+    if (NOT APPLE)
+      set(FS_STDLIB stdc++fs)
+    endif()
+  endif()
+  list(APPEND onnxruntime_EXTERNAL_LIBRARIES tvm nnvm_compiler ${FS_STDLIB})
   list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES tvm nnvm_compiler)
 endif()
 

diff --git a/cmake/external/tvm b/cmake/external/tvm
diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake
@@ -63,6 +63,7 @@ target_link_libraries(onnxruntime PRIVATE
     ${PROVIDERS_NNAPI}
     ${PROVIDERS_TENSORRT}
     ${PROVIDERS_OPENVINO}
+    ${PROVIDERS_NUPHAR}
     onnxruntime_optimizer
     onnxruntime_providers
     onnxruntime_util

diff --git a/cmake/onnxruntime_nuphar_extern.cmake b/cmake/onnxruntime_nuphar_extern.cmake
@@ -0,0 +1,39 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# this is for building extern functions in nuphar execution provider, using AVX2
+# the separation from onnxruntime_providers.cmake is to avoid unnecessary AVX2 codegen in providers
+# functions built here would be dynamically switched based on if AVX2 is available from CPUID
+
+add_definitions(-DNUPHAR_USE_AVX2)
+
+set(extern_avx2_srcs
+  ${ONNXRUNTIME_ROOT}/core/providers/nuphar/extern/igemv_avx2.cc
+  ${ONNXRUNTIME_ROOT}/core/providers/nuphar/extern/igemv_avx2.h
+)
+
+if (MSVC) 
+  set_source_files_properties(${extern_avx2_srcs} PROPERTIES COMPILE_FLAGS "/arch:AVX2")
+else()
+  set_source_files_properties(${extern_avx2_srcs} PROPERTIES COMPILE_FLAGS "-march=broadwell")
+endif()
+
+set(nuphar_extern_srcs
+    ${extern_avx2_srcs}
+)
+
+add_library(onnxruntime_nuphar_extern  ${nuphar_extern_srcs})
+
+if (onnxruntime_USE_MKLML)
+  add_definitions(-DNUPHAR_USE_MKL)
+  target_include_directories(onnxruntime_nuphar_extern PRIVATE ${ONNXRUNTIME_ROOT}/core/providers/nuphar/extern ${MKLML_INCLUDE_DIR})
+  add_dependencies(onnxruntime_nuphar_extern project_mklml)
+else()
+  target_include_directories(onnxruntime_nuphar_extern PRIVATE ${ONNXRUNTIME_ROOT}/core/providers/nuphar/extern)
+endif()
+
+set_target_properties(onnxruntime_nuphar_extern PROPERTIES FOLDER "ONNXRuntime")
+
+list(APPEND onnxruntime_EXTERNAL_LIBRARIES onnxruntime_nuphar_extern)
+list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES onnxruntime_nuphar_extern)
+link_directories(${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE})
diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake
@@ -48,6 +48,10 @@ if(onnxruntime_USE_NGRAPH)
   set(PROVIDERS_NGRAPH onnxruntime_providers_ngraph)
   list(APPEND ONNXRUNTIME_PROVIDER_NAMES ngraph)
 endif()
+if(onnxruntime_USE_NUPHAR)
+  set(PROVIDERS_NUPHAR onnxruntime_providers_nuphar)
+  list(APPEND ONNXRUNTIME_PROVIDER_NAMES nuphar)
+endif()
 if(onnxruntime_USE_CUDA)
   set(PROVIDERS_CUDA onnxruntime_providers_cuda)
   list(APPEND ONNXRUNTIME_PROVIDER_NAMES cuda)
@@ -318,6 +322,43 @@ endif()
   file(COPY ${onnxruntime_providers_openvino_py_srcs} DESTINATION ${onnxruntime_BINARY_DIR})
 endif()
 
+if (onnxruntime_USE_NUPHAR)
+  add_definitions(-DUSE_NUPHAR=1)
+
+  if (NOT onnxruntime_USE_TVM)
+    message(FATAL_ERROR "onnxruntime_USE_TVM required for onnxruntime_USE_NUPHAR")
+  endif()
+
+  if (NOT onnxruntime_USE_LLVM)
+    message(FATAL_ERROR "onnxruntime_USE_LLVM required for onnxruntime_USE_NUPHAR")
+  endif()
+
+  include(onnxruntime_nuphar_extern.cmake)
+
+  file(GLOB_RECURSE onnxruntime_providers_nuphar_cc_srcs
+    "${ONNXRUNTIME_ROOT}/core/providers/nuphar/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/nuphar/*.cc"
+  )
+
+  # following files required different build flag for AVX2 in separate onnxruntime_nuphar_extern.cmake file
+  list (REMOVE_ITEM onnxruntime_providers_nuphar_cc_srcs "${ONNXRUNTIME_ROOT}/core/providers/nuphar/extern/igemv_avx2.cc")
+  list (REMOVE_ITEM onnxruntime_providers_nuphar_cc_srcs "${ONNXRUNTIME_ROOT}/core/providers/nuphar/extern/igemv_avx2.h")
+
+  if (onnxruntime_USE_MKLML)
+    add_definitions(-DNUPHAR_USE_MKL)
+  endif()
+
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_nuphar_cc_srcs})
+  add_library(onnxruntime_providers_nuphar ${onnxruntime_providers_nuphar_cc_srcs})
+  onnxruntime_add_include_to_target(onnxruntime_providers_nuphar onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
+  set_target_properties(onnxruntime_providers_nuphar PROPERTIES FOLDER "ONNXRuntime")
+  target_include_directories(onnxruntime_providers_nuphar PRIVATE ${ONNXRUNTIME_ROOT} ${TVM_INCLUDES} ${eigen_INCLUDE_DIRS})
+  set_target_properties(onnxruntime_providers_nuphar PROPERTIES LINKER_LANGUAGE CXX)
+  target_compile_options(onnxruntime_providers_nuphar PRIVATE ${DISABLED_WARNINGS_FOR_TVM})
+  add_dependencies(onnxruntime_providers_nuphar ${onnxruntime_EXTERNAL_DEPENDENCIES})
+  install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/nuphar  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
+endif()
+
 if (onnxruntime_USE_NNAPI)
   add_definitions(-DUSE_NNAPI=1)
   option(DNN_READ_ONNX "" ON)

diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake
@@ -73,6 +73,7 @@ set(onnxruntime_pybind11_state_libs
     ${PROVIDERS_TENSORRT}
     ${PROVIDERS_NGRAPH}
     ${PROVIDERS_OPENVINO}
+    ${PROVIDERS_NUPHAR}
     ${PROVIDERS_NNAPI}
     onnxruntime_optimizer
     onnxruntime_providers
@@ -234,3 +235,15 @@ if (onnxruntime_USE_MKLML)
         $<TARGET_FILE_DIR:${test_data_target}>/onnxruntime/capi/
   )
 endif()
+
+if (onnxruntime_USE_NUPHAR)
+  file(GLOB onnxruntime_python_nuphar_test_srcs CONFIGURE_DEPENDS
+    "${ONNXRUNTIME_ROOT}/core/providers/nuphar/scripts/*.*"
+  )
+  add_custom_command(
+    TARGET onnxruntime_pybind11_state POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy
+      ${onnxruntime_python_nuphar_test_srcs}
+      $<TARGET_FILE_DIR:${test_data_target}>
+  )
+endif()
diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
@@ -229,6 +229,13 @@ file(GLOB_RECURSE onnxruntime_test_openvino_src
   "${ONNXRUNTIME_ROOT}/test/openvino/*.cc"
  )
 
+if(onnxruntime_USE_NUPHAR)
+  list(APPEND onnxruntime_test_framework_src_patterns  ${TEST_SRC_DIR}/framework/nuphar/*)
+  list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_nuphar)
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_nuphar)
+  list(APPEND onnxruntime_test_providers_libs onnxruntime_providers_nuphar)
+endif()
+
 if (onnxruntime_ENABLE_MICROSOFT_INTERNAL)
   include(onnxruntime_unittests_internal.cmake)
 endif()
@@ -241,6 +248,7 @@ set(ONNXRUNTIME_TEST_LIBS
     ${PROVIDERS_TENSORRT}
     ${PROVIDERS_NGRAPH}
     ${PROVIDERS_OPENVINO}
+    ${PROVIDERS_NUPHAR}
     ${PROVIDERS_NNAPI}
     onnxruntime_optimizer
     onnxruntime_providers
@@ -520,6 +528,12 @@ onnxruntime_add_include_to_target(onnx_test_runner gsl)
 target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT})
 set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest")
 
+if (onnxruntime_USE_TVM)
+  if (WIN32)
+    target_link_options(onnx_test_runner PRIVATE "/STACK:4000000")
+  endif()
+endif()
+
 install(TARGETS onnx_test_runner
         ARCHIVE  DESTINATION ${CMAKE_INSTALL_LIBDIR}
         LIBRARY  DESTINATION ${CMAKE_INSTALL_LIBDIR}
@@ -600,6 +614,12 @@ if (onnxruntime_ENABLE_LANGUAGE_INTEROP_OPS AND NOT onnxruntime_BUILD_SHARED_LIB
   target_link_libraries(onnxruntime_perf_test PRIVATE onnxruntime_language_interop onnxruntime_pyop)
 endif()
 
+if (onnxruntime_USE_TVM)
+  if (WIN32)
+    target_link_options(onnxruntime_perf_test PRIVATE "/STACK:4000000")
+  endif()
+endif()
+
 # shared lib
 if (onnxruntime_BUILD_SHARED_LIB)
   add_library(onnxruntime_mocked_allocator ${ONNXRUNTIME_ROOT}/test/util/test_allocator.cc)

diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj b/csharp/src/Microsoft.ML.OnnxRuntime/Microsoft.ML.OnnxRuntime.csproj
@@ -99,6 +99,55 @@
           CopyToOutputDirectory="Always"
           Visible="false"
     />
+    <None Include="$(NativeBuildOutputDir)\tvm.dll"
+          Condition="Exists('$(NativeBuildOutputDir)\tvm.dll')"
+          PackagePath="\runtimes\win-$(TargetArchitecture)\native"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\create_shared.cmd"
+          Condition="Exists('$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\create_shared.cmd')"
+          PackagePath="\scripts"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\create_shared.sh"
+          Condition="Exists('$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\create_shared.sh')"
+          PackagePath="\scripts"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\model_editor.py"
+          Condition="Exists('$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\model_editor.py')"
+          PackagePath="\scripts"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\model_quantizer.py"
+          Condition="Exists('$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\model_quantizer.py')"
+          PackagePath="\scripts"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\node_factory.py"
+          Condition="Exists('$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\node_factory.py')"
+          PackagePath="\scripts"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
+    <None Include="$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\symbolic_shape_infer.py"
+          Condition="Exists('$(OnnxRuntimeCsharpRoot)\..\onnxruntime\core\providers\nuphar\scripts\symbolic_shape_infer.py')"
+          PackagePath="\scripts"
+          Pack="true"
+          CopyToOutputDirectory="Always"
+          Visible="false"
+    />
     <None Include="$(OnnxRuntimeCsharpRoot)\..\LICENSE.txt;$(OnnxRuntimeCsharpRoot)\..\ThirdPartyNotices.txt"
           PackagePath="\"
           Pack="true"

diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs
@@ -181,8 +181,8 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_CUDA(IntPtr /*(OrtSessionOptions*) */ options, int device_id);
 
-        //[DllImport(nativeLib, CharSet = charSet)]
-        //public static extern IntPtr /*(OrtStatus*)*/ OrtCreateNupharExecutionProviderFactory(int device_id, string target_str, out IntPtr /*(OrtProviderFactoryPtr**)*/ factory);
+        [DllImport(nativeLib, CharSet = charSet)]
+        public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Nuphar(IntPtr /*(OrtSessionOptions*) */ options, int allow_unaligned_buffers, string settings);
 
         //[DllImport(nativeLib, CharSet = charSet)]
         //public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path);

diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs
@@ -62,6 +62,18 @@ public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId = 0)
             return options;
         }
 
+        /// <summary>
+        /// A helper method to construct a SessionOptions object for Nuphar execution
+        /// </summary>
+        /// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
+        /// <returns>A SessionsOptions() object configured for execution with Nuphar</returns>
+        public static SessionOptions MakeSessionOptionWithNupharProvider(String settings = "")
+        {
+            SessionOptions options = new SessionOptions();
+            NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(options._nativePtr, 1, settings);
+            return options;
+        }
+
         #endregion
 
         #region Public Properties
+1 −1		cmake/patches/halide_fp_simplify.patch
+7 −0		include/tvm/buffer.h
+6 −0		include/tvm/expr.h
+3 −3		include/tvm/ir_pass.h
+36 −1		src/codegen/llvm/codegen_llvm.cc
+21 −0		src/lang/buffer.cc
+8 −0		src/lang/expr.cc
+18 −10		src/op/compute_op.cc
+3 −1		src/op/compute_op.h
+3 −1		src/op/tensorize.cc