From b03dc6faa2ed3c025765380ca19bf2178833afde Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Mon, 22 Jul 2019 11:09:12 -0700 Subject: [PATCH 1/8] Mention OrtCreateSessionFromArray in C API doc --- docs/C_API.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/C_API.md b/docs/C_API.md index e13ddecfb095e..ea99c5875fe45 100644 --- a/docs/C_API.md +++ b/docs/C_API.md @@ -11,6 +11,7 @@ * Setting the thread pool size for each session. * Setting graph optimization level for each session. * Dynamically loading custom ops. [Instructions](/docs/AddingCustomOp.md) +* Ability to load a model from a byte array. See ```OrtCreateSessionFromArray``` in [onnxruntime_c_api.h](/include/onnxruntime/core/session/onnxruntime_c_api.h). ## Usage Overview From dcd698215a990d2f09d0ef3d9fdf7739d75da678 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Tue, 13 Aug 2019 19:09:13 -0700 Subject: [PATCH 2/8] Don't create the default allocator every single time. Rename API accordingly. --- .../DisposableNamedOnnxValue.cs | 3 +-- .../NativeMemoryAllocator.cs | 19 ++++--------------- .../Microsoft.ML.OnnxRuntime/NativeMethods.cs | 5 +---- .../CXX_Api_Sample.cpp | 2 +- .../C_Api_Sample.cpp | 3 +-- .../InferenceTest.cs | 2 +- .../core/session/onnxruntime_c_api.h | 2 +- .../core/session/onnxruntime_cxx_api.h | 3 +-- .../core/session/onnxruntime_cxx_inline.h | 4 ++-- onnxruntime/core/providers/cpu/symbols.txt | 3 +-- .../session/default_cpu_allocator_c_api.cc | 9 +++------ onnxruntime/server/environment.cc | 2 +- onnxruntime/test/perftest/ort_test_session.cc | 2 +- .../test/server/unit_tests/converter_tests.cc | 2 +- onnxruntime/test/shared_lib/test_allocator.cc | 2 +- samples/c_cxx/imagenet/main.cc | 5 ++--- 16 files changed, 23 insertions(+), 45 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs index 096501019771e..ebf3a7e7839e3 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs @@ -134,9 +134,8 @@ internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name, internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) { IntPtr allocator = IntPtr.Zero; - NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateDefaultAllocator(out allocator)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDefaultAllocator(out allocator)); var ret = CreateFromOnnxValue(name, nativeOnnxValue, allocator); - NativeMethods.OrtReleaseAllocator(allocator); return (DisposableNamedOnnxValue)ret; } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs index a9b4e60f5ac58..a412316561dfa 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs @@ -77,22 +77,18 @@ protected override bool ReleaseHandle() internal class NativeMemoryAllocator : SafeHandle { - protected static readonly Lazy _defaultInstance = new Lazy(CreateDefaultCpuAllocator); + protected static readonly Lazy _defaultInstance = new Lazy(GetDefaultCpuAllocator); - private static NativeMemoryAllocator CreateDefaultCpuAllocator() + private static NativeMemoryAllocator GetDefaultCpuAllocator() { IntPtr allocator = IntPtr.Zero; try { - IntPtr status = NativeMethods.OrtCreateDefaultAllocator(out allocator); + IntPtr status = NativeMethods.OrtGetDefaultAllocator(out allocator); NativeApiStatus.VerifySuccess(status); } catch (Exception e) { - if (allocator != IntPtr.Zero) - { - Delete(allocator); - } throw e; } @@ -124,7 +120,7 @@ public override bool IsInvalid } } - internal IntPtr Handle + internal IntPtr Handle { get { @@ -138,15 +134,8 @@ protected NativeMemoryAllocator(IntPtr allocator) this.handle = allocator; } - - protected static void Delete(IntPtr allocator) - { - NativeMethods.OrtReleaseAllocator(allocator); - } - protected override bool ReleaseHandle() { - Delete(this.handle); return true; } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 140516a6302f4..8655e8fd522bb 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -226,10 +226,7 @@ public enum MemoryType public static extern void OrtReleaseAllocatorInfo(IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(OrtStatus*)*/OrtCreateDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); - - [DllImport(nativeLib, CharSet = charSet)] - public static extern void OrtReleaseAllocator(IntPtr /*(OrtAllocator*)*/ allocator); + public static extern IntPtr /*(OrtStatus*)*/OrtGetDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); /// /// Release any object allocated by an allocator diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp index 34765ab133fa2..13c1f9a5a04f4 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp @@ -43,7 +43,7 @@ int main(int argc, char* argv[]) { //************************************************************************* // print model input layer (node names, types, shape etc.) - Ort::Allocator allocator = Ort::Allocator::CreateDefault(); + Ort::Allocator allocator = Ort::Allocator::GetDefault(); // print number of model input nodes size_t num_input_nodes = session.GetInputCount(); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp index 11dae1ab52197..66c73b54ae068 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp @@ -63,7 +63,7 @@ int main(int argc, char* argv[]) { size_t num_input_nodes; OrtStatus* status; OrtAllocator* allocator; - CHECK_STATUS(OrtCreateDefaultAllocator(&allocator)); + CHECK_STATUS(OrtGetDefaultAllocator(&allocator)); // print number of model input nodes status = OrtSessionGetInputCount(session, &num_input_nodes); @@ -101,7 +101,6 @@ int main(int argc, char* argv[]) { OrtReleaseTypeInfo(typeinfo); } - OrtReleaseAllocator(allocator); // Results should be... // Number of inputs = 1 diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 471d7fb267ff0..d85c2b65ae0ba 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -693,7 +693,7 @@ private void VerifyNativeMethodsExist() "OrtEnableMemPattern","OrtDisableMemPattern","OrtEnableCpuMemArena","OrtDisableCpuMemArena", "OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel", "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU","OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo", - "OrtCreateDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo", + "OrtGetDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo", "OrtCreateTensorWithDataAsOrtValue","OrtGetTensorMutableData", "OrtReleaseAllocatorInfo", "OrtCastTypeInfoToTensorInfo","OrtGetTensorTypeAndShape","OrtGetTensorElementType","OrtGetDimensionsCount", "OrtGetDimensions","OrtGetTensorShapeElementCount","OrtReleaseValue"}; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index f1cd4638c48d7..5901fb3da5e08 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -434,7 +434,7 @@ ORT_API_STATUS(OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outpt ORT_API_STATUS(OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); ORT_API_STATUS(OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr, _Out_ const OrtAllocatorInfo** out); -ORT_API_STATUS(OrtCreateDefaultAllocator, _Outptr_ OrtAllocator** out); +ORT_API_STATUS(OrtGetDefaultAllocator, _Outptr_ OrtAllocator** out); ORT_API(const char*, OrtGetVersionString); /** diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index f5757c79403b0..8143a65625828 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -43,7 +43,6 @@ struct Exception : std::exception { #define ORT_DEFINE_RELEASE(NAME) \ inline void OrtRelease(Ort##NAME* ptr) { OrtRelease##NAME(ptr); } -ORT_DEFINE_RELEASE(Allocator); ORT_DEFINE_RELEASE(AllocatorInfo); ORT_DEFINE_RELEASE(CustomOpDomain); ORT_DEFINE_RELEASE(Env); @@ -228,7 +227,7 @@ struct Value : Base { }; struct Allocator : Base { - static Allocator CreateDefault(); + static Allocator GetDefault(); explicit Allocator(nullptr_t) {} explicit Allocator(OrtAllocator* p) : Base{p} {} diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 9694603ad197d..7b97fd7b4a222 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -39,9 +39,9 @@ struct TypeToTensorType { static constexpr ONNXTensorElementDataType t template <> struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; -inline Allocator Allocator::CreateDefault() { +inline Allocator Allocator::GetDefault() { OrtAllocator* p; - ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&p)); + ORT_THROW_ON_ERROR(OrtGetDefaultAllocator(&p)); return Allocator(p); } diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index e6324e2b6d519..211e732fcd18e 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -12,7 +12,7 @@ OrtCompareAllocatorInfo OrtCreateAllocatorInfo OrtCreateCpuAllocatorInfo OrtCreateCustomOpDomain -OrtCreateDefaultAllocator +OrtGetDefaultAllocator OrtCreateEnv OrtCreateEnvWithCustomLogger OrtCreateRunOptions @@ -51,7 +51,6 @@ OrtGetValueType OrtGetVersionString OrtIsTensor OrtGetOnnxTypeFromTypeInfo -OrtReleaseAllocator OrtReleaseAllocatorInfo OrtReleaseCustomOpDomain OrtReleaseEnv diff --git a/onnxruntime/core/session/default_cpu_allocator_c_api.cc b/onnxruntime/core/session/default_cpu_allocator_c_api.cc index a4a7e7ea51612..eb5f3a97b3d79 100644 --- a/onnxruntime/core/session/default_cpu_allocator_c_api.cc +++ b/onnxruntime/core/session/default_cpu_allocator_c_api.cc @@ -46,13 +46,10 @@ struct OrtDefaultAllocator : OrtAllocatorImpl { return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } -ORT_API_STATUS_IMPL(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out) { +ORT_API_STATUS_IMPL(OrtGetDefaultAllocator, _Out_ OrtAllocator** out) { API_IMPL_BEGIN - *out = new OrtDefaultAllocator(); + static OrtDefaultAllocator ort_default_allocator; + *out = &ort_default_allocator; return nullptr; API_IMPL_END } - -ORT_API(void, OrtReleaseAllocator, _In_ OrtAllocator* allocator) { - delete static_cast(allocator); -} diff --git a/onnxruntime/server/environment.cc b/onnxruntime/server/environment.cc index 6ebcb90b4cef5..9888ecba6e6ef 100644 --- a/onnxruntime/server/environment.cc +++ b/onnxruntime/server/environment.cc @@ -48,7 +48,7 @@ void ServerEnvironment::InitializeModel(const std::string& model_path) { auto output_count = session.GetOutputCount(); - auto allocator = Ort::Allocator::CreateDefault(); + auto allocator = Ort::Allocator::GetDefault(); for (size_t i = 0; i < output_count; i++) { auto name = session.GetOutputName(i, allocator); model_output_names_.push_back(name); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index d74fb1ba9ad32..680261c9c05cf 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -102,7 +102,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device size_t output_count = session_.GetOutputCount(); output_names_.resize(output_count); - Ort::Allocator a = Ort::Allocator::CreateDefault(); + Ort::Allocator a = Ort::Allocator::GetDefault(); for (size_t i = 0; i != output_count; ++i) { char* output_name = session_.GetOutputName(i, a); assert(output_name != nullptr); diff --git a/onnxruntime/test/server/unit_tests/converter_tests.cc b/onnxruntime/test/server/unit_tests/converter_tests.cc index 4b192dd0d08c7..d36f76c3ed2ce 100644 --- a/onnxruntime/test/server/unit_tests/converter_tests.cc +++ b/onnxruntime/test/server/unit_tests/converter_tests.cc @@ -304,7 +304,7 @@ TEST(MLValueToTensorProtoTests, UInt8ProtoRoundTrip) { tp.set_data_type(onnx::TensorProto_DataType_UINT8); Ort::Value ml_value{nullptr}; char buf[1000]; - auto allocator = Ort::Allocator::CreateDefault(); + auto allocator = Ort::Allocator::GetDefault(); auto info = allocator.GetInfo(); MemBuffer buffer((void*)&buf, tp.ByteSizeLong(), *info); onnxruntime::server::TensorProtoToMLValue(tp, buffer, ml_value); diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc index 372594fa8f16c..cfebc23f307d1 100644 --- a/onnxruntime/test/shared_lib/test_allocator.cc +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -20,7 +20,7 @@ TEST_F(CApiTest, allocation_info) { } TEST_F(CApiTest, DefaultAllocator) { - Ort::Allocator default_allocator = Ort::Allocator::CreateDefault(); + Ort::Allocator default_allocator = Ort::Allocator::GetDefault(); char* p = (char*)default_allocator.Alloc(100); ASSERT_NE(p, nullptr); memset(p, 0, 100); diff --git a/samples/c_cxx/imagenet/main.cc b/samples/c_cxx/imagenet/main.cc index 1077bcd40789f..89ce9cfcccd89 100644 --- a/samples/c_cxx/imagenet/main.cc +++ b/samples/c_cxx/imagenet/main.cc @@ -128,7 +128,7 @@ class Validator : public OutputCollector { CreateSession(); VerifyInputOutputCount(session_); OrtAllocator* ort_alloc; - ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&ort_alloc)); + ORT_THROW_ON_ERROR(OrtGetDefaultAllocator(&ort_alloc)); { char* t; ORT_THROW_ON_ERROR(OrtSessionGetInputName(session_, 0, ort_alloc, &t)); @@ -139,7 +139,6 @@ class Validator : public OutputCollector { OrtAllocatorFree(ort_alloc, t); } - OrtReleaseAllocator(ort_alloc); OrtTypeInfo* info; ORT_THROW_ON_ERROR(OrtSessionGetInputTypeInfo(session_, 0, &info)); const OrtTensorTypeAndShapeInfo* tensor_info; @@ -253,7 +252,7 @@ int main(int argc, ORTCHAR_T* argv[]) { try { ret = real_main(argc, argv); } catch (const std::exception& ex) { - fprintf(stderr, "%s\n", ex.what()); + fprintf(stderr, "%s\n", ex.what()); } #ifdef _WIN32 CoUninitialize(); From 723ded7b5f494086e29b4961190c15e3ef87e556 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Tue, 13 Aug 2019 19:09:13 -0700 Subject: [PATCH 3/8] Don't create the default allocator every single time. Rename API accordingly. --- .../DisposableNamedOnnxValue.cs | 2 +- .../Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs | 9 ++++----- csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs | 7 +++++-- .../CXX_Api_Sample.cpp | 2 +- .../C_Api_Sample.cpp | 2 +- .../Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs | 4 ++-- include/onnxruntime/core/session/onnxruntime_c_api.h | 7 +++++-- include/onnxruntime/core/session/onnxruntime_cxx_api.h | 5 ++++- .../onnxruntime/core/session/onnxruntime_cxx_inline.h | 9 +++++++-- onnxruntime/core/framework/run_options.cc | 10 ++++++++++ onnxruntime/core/providers/cpu/symbols.txt | 4 +++- onnxruntime/core/session/abi_session_options.cc | 5 +++++ .../core/session/default_cpu_allocator_c_api.cc | 5 +++-- onnxruntime/server/environment.cc | 2 +- onnxruntime/test/perftest/ort_test_session.cc | 2 +- onnxruntime/test/server/unit_tests/converter_tests.cc | 2 +- onnxruntime/test/shared_lib/test_allocator.cc | 2 +- samples/c_cxx/imagenet/main.cc | 4 ++-- 18 files changed, 57 insertions(+), 26 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs index 096501019771e..1391a03d13bf5 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs @@ -134,7 +134,7 @@ internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name, internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) { IntPtr allocator = IntPtr.Zero; - NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateDefaultAllocator(out allocator)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDefaultAllocator(out allocator)); var ret = CreateFromOnnxValue(name, nativeOnnxValue, allocator); NativeMethods.OrtReleaseAllocator(allocator); return (DisposableNamedOnnxValue)ret; diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs index a9b4e60f5ac58..087a3eebc0ccf 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs @@ -77,14 +77,14 @@ protected override bool ReleaseHandle() internal class NativeMemoryAllocator : SafeHandle { - protected static readonly Lazy _defaultInstance = new Lazy(CreateDefaultCpuAllocator); + protected static readonly Lazy _defaultInstance = new Lazy(GetDefaultCpuAllocator); - private static NativeMemoryAllocator CreateDefaultCpuAllocator() + private static NativeMemoryAllocator GetDefaultCpuAllocator() { IntPtr allocator = IntPtr.Zero; try { - IntPtr status = NativeMethods.OrtCreateDefaultAllocator(out allocator); + IntPtr status = NativeMethods.OrtGetDefaultAllocator(out allocator); NativeApiStatus.VerifySuccess(status); } catch (Exception e) @@ -124,7 +124,7 @@ public override bool IsInvalid } } - internal IntPtr Handle + internal IntPtr Handle { get { @@ -138,7 +138,6 @@ protected NativeMemoryAllocator(IntPtr allocator) this.handle = allocator; } - protected static void Delete(IntPtr allocator) { NativeMethods.OrtReleaseAllocator(allocator); diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 140516a6302f4..ed5f8d9aa8ae6 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -157,6 +157,9 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtSetSessionLogVerbosityLevel(IntPtr /* OrtSessionOptions* */ options, LogLevel sessionLogVerbosityLevel); + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtSetSessionLogSeverityLevel(IntPtr /* OrtSessionOptions* */ options, LogLevel sessionLogSeverityLevel); + [DllImport(nativeLib, CharSet = charSet)] public static extern IntPtr /*(OrtStatus*)*/ OrtSetSessionThreadPoolSize(IntPtr /* OrtSessionOptions* */ options, int sessionThreadPoolSize); @@ -226,10 +229,10 @@ public enum MemoryType public static extern void OrtReleaseAllocatorInfo(IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(OrtStatus*)*/OrtCreateDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); + public static extern void OrtReleaseAllocator(IntPtr /*(OrtAllocator*)*/ allocator); [DllImport(nativeLib, CharSet = charSet)] - public static extern void OrtReleaseAllocator(IntPtr /*(OrtAllocator*)*/ allocator); + public static extern IntPtr /*(OrtStatus*)*/OrtGetDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); /// /// Release any object allocated by an allocator diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp index 34765ab133fa2..13c1f9a5a04f4 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp @@ -43,7 +43,7 @@ int main(int argc, char* argv[]) { //************************************************************************* // print model input layer (node names, types, shape etc.) - Ort::Allocator allocator = Ort::Allocator::CreateDefault(); + Ort::Allocator allocator = Ort::Allocator::GetDefault(); // print number of model input nodes size_t num_input_nodes = session.GetInputCount(); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp index 11dae1ab52197..12ccb55c4c739 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp @@ -63,7 +63,7 @@ int main(int argc, char* argv[]) { size_t num_input_nodes; OrtStatus* status; OrtAllocator* allocator; - CHECK_STATUS(OrtCreateDefaultAllocator(&allocator)); + CHECK_STATUS(OrtGetDefaultAllocator(&allocator)); // print number of model input nodes status = OrtSessionGetInputCount(session, &num_input_nodes); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 471d7fb267ff0..08ee897c8ef53 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -691,9 +691,9 @@ private void VerifyNativeMethodsExist() "OrtSessionGetOutputTypeInfo","OrtReleaseSession","OrtCreateSessionOptions","OrtCloneSessionOptions", "OrtEnableSequentialExecution","OrtDisableSequentialExecution","OrtEnableProfiling","OrtDisableProfiling", "OrtEnableMemPattern","OrtDisableMemPattern","OrtEnableCpuMemArena","OrtDisableCpuMemArena", - "OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel", + "OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionLogSeverityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel", "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU","OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo", - "OrtCreateDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo", + "OrtGetDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo", "OrtCreateTensorWithDataAsOrtValue","OrtGetTensorMutableData", "OrtReleaseAllocatorInfo", "OrtCastTypeInfoToTensorInfo","OrtGetTensorTypeAndShape","OrtGetTensorElementType","OrtGetDimensionsCount", "OrtGetDimensions","OrtGetTensorShapeElementCount","OrtReleaseValue"}; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index f1cd4638c48d7..39f7dc5b73c10 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -202,7 +202,7 @@ ORT_API_STATUS(OrtRun, _Inout_ OrtSession* sess, ORT_API_STATUS(OrtCreateSessionOptions, _Outptr_ OrtSessionOptions** options); // Set filepath to save optimized model after graph level transformations. -ORT_API_STATUS(OrtSetOptimizedModelFilePath, _In_ OrtSessionOptions* options, _In_ const ORTCHAR_T* optimized_model_filepath); +ORT_API_STATUS(OrtSetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* optimized_model_filepath); // create a copy of an existing OrtSessionOptions ORT_API_STATUS(OrtCloneSessionOptions, _In_ const OrtSessionOptions* in_options, _Outptr_ OrtSessionOptions** out_options); @@ -232,6 +232,7 @@ ORT_API_STATUS(OrtSetSessionLogId, _Inout_ OrtSessionOptions* options, const cha // < applies to session load, initialization, etc ORT_API_STATUS(OrtSetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level); +ORT_API_STATUS(OrtSetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level); // Set Graph optimization level. // Available options are : 0, 1, 2. @@ -281,9 +282,11 @@ ORT_API_STATUS(OrtSessionGetOutputName, _In_ const OrtSession* sess, size_t inde ORT_API_STATUS(OrtCreateRunOptions, _Outptr_ OrtRunOptions** out); ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value); +ORT_API_STATUS(OrtRunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value); ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag); ORT_API_STATUS(OrtRunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out); +ORT_API_STATUS(OrtRunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out); ORT_API_STATUS(OrtRunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out); // Set a flag so that any running OrtRun* calls that are using this instance of OrtRunOptions @@ -434,7 +437,7 @@ ORT_API_STATUS(OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outpt ORT_API_STATUS(OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); ORT_API_STATUS(OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr, _Out_ const OrtAllocatorInfo** out); -ORT_API_STATUS(OrtCreateDefaultAllocator, _Outptr_ OrtAllocator** out); +ORT_API_STATUS(OrtGetDefaultAllocator, _Outptr_ OrtAllocator** out); ORT_API(const char*, OrtGetVersionString); /** diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index f5757c79403b0..6edde2c40aa08 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -120,6 +120,9 @@ struct RunOptions : Base { RunOptions& SetRunLogVerbosityLevel(int); int GetRunLogVerbosityLevel() const; + RunOptions& SetRunLogSeverityLevel(int); + int GetRunLogSeverityLevel() const; + RunOptions& SetRunTag(const char* run_tag); const char* GetRunTag() const; @@ -228,7 +231,7 @@ struct Value : Base { }; struct Allocator : Base { - static Allocator CreateDefault(); + static Allocator GetDefault(); explicit Allocator(nullptr_t) {} explicit Allocator(OrtAllocator* p) : Base{p} {} diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 9694603ad197d..8b86923d80e3a 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -39,9 +39,9 @@ struct TypeToTensorType { static constexpr ONNXTensorElementDataType t template <> struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; -inline Allocator Allocator::CreateDefault() { +inline Allocator Allocator::GetDefault() { OrtAllocator* p; - ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&p)); + ORT_THROW_ON_ERROR(OrtGetDefaultAllocator(&p)); return Allocator(p); } @@ -96,6 +96,11 @@ inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) { return *this; } +inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) { + ORT_THROW_ON_ERROR(OrtRunOptionsSetRunLogSeverityLevel(p_, level)); + return *this; +} + inline int RunOptions::GetRunLogVerbosityLevel() const { int out; ORT_THROW_ON_ERROR(OrtRunOptionsGetRunLogVerbosityLevel(p_, &out)); diff --git a/onnxruntime/core/framework/run_options.cc b/onnxruntime/core/framework/run_options.cc index 079be56fc5ae4..640c610841774 100644 --- a/onnxruntime/core/framework/run_options.cc +++ b/onnxruntime/core/framework/run_options.cc @@ -17,6 +17,11 @@ ORT_API_STATUS_IMPL(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions* op return nullptr; } +ORT_API_STATUS_IMPL(OrtRunOptionsSetRunLogSeverityLevel, _In_ OrtRunOptions* options, int value) { + options->run_log_severity_level = value; + return nullptr; +} + ORT_API_STATUS_IMPL(OrtRunOptionsSetRunTag, _In_ OrtRunOptions* options, _In_ const char* run_tag) { if (run_tag) options->run_tag = run_tag; @@ -28,6 +33,11 @@ ORT_API_STATUS_IMPL(OrtRunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptio return nullptr; } +ORT_API_STATUS_IMPL(OrtRunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, int* out) { + *out = options->run_log_severity_level; + return nullptr; +} + ORT_API_STATUS_IMPL(OrtRunOptionsGetRunTag, _In_ const OrtRunOptions* options, const char** out) { *out = options->run_tag.c_str(); return nullptr; diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index e6324e2b6d519..c00ef47f04e95 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -12,7 +12,7 @@ OrtCompareAllocatorInfo OrtCreateAllocatorInfo OrtCreateCpuAllocatorInfo OrtCreateCustomOpDomain -OrtCreateDefaultAllocator +OrtGetDefaultAllocator OrtCreateEnv OrtCreateEnvWithCustomLogger OrtCreateRunOptions @@ -66,6 +66,7 @@ OrtRun OrtRunOptionsGetRunLogVerbosityLevel OrtRunOptionsGetRunTag OrtRunOptionsSetRunLogVerbosityLevel +OrtRunOptionsSetRunLogSeverityLevel OrtRunOptionsSetRunTag OrtRunOptionsEnableTerminate OrtRunOptionsDisableTerminate @@ -80,6 +81,7 @@ OrtSetDimensions OrtSetSessionGraphOptimizationLevel OrtSetSessionLogId OrtSetSessionLogVerbosityLevel +OrtSetSessionLogSeverityLevel OrtSetOptimizedModelFilePath OrtSetSessionThreadPoolSize OrtSetTensorElementType diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index a3be9e8f59853..46a1a253c6da4 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -100,6 +100,11 @@ ORT_API_STATUS_IMPL(OrtSetSessionLogVerbosityLevel, _In_ OrtSessionOptions* opti return nullptr; } +ORT_API_STATUS_IMPL(OrtSetSessionLogSeverityLevel, _In_ OrtSessionOptions* options, int session_log_severity_level) { + options->value.session_log_severity_level = session_log_severity_level; + return nullptr; +} + // Set Graph optimization level. // Available options are : 0, 1, 2. ORT_API_STATUS_IMPL(OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions* options, int graph_optimization_level) { diff --git a/onnxruntime/core/session/default_cpu_allocator_c_api.cc b/onnxruntime/core/session/default_cpu_allocator_c_api.cc index a4a7e7ea51612..4e0b26eca51b0 100644 --- a/onnxruntime/core/session/default_cpu_allocator_c_api.cc +++ b/onnxruntime/core/session/default_cpu_allocator_c_api.cc @@ -46,9 +46,10 @@ struct OrtDefaultAllocator : OrtAllocatorImpl { return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } -ORT_API_STATUS_IMPL(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out) { +ORT_API_STATUS_IMPL(OrtGetDefaultAllocator, _Out_ OrtAllocator** out) { API_IMPL_BEGIN - *out = new OrtDefaultAllocator(); + static OrtDefaultAllocator* ort_default_allocator = new OrtDefaultAllocator(); + *out = ort_default_allocator; return nullptr; API_IMPL_END } diff --git a/onnxruntime/server/environment.cc b/onnxruntime/server/environment.cc index 6ebcb90b4cef5..9888ecba6e6ef 100644 --- a/onnxruntime/server/environment.cc +++ b/onnxruntime/server/environment.cc @@ -48,7 +48,7 @@ void ServerEnvironment::InitializeModel(const std::string& model_path) { auto output_count = session.GetOutputCount(); - auto allocator = Ort::Allocator::CreateDefault(); + auto allocator = Ort::Allocator::GetDefault(); for (size_t i = 0; i < output_count; i++) { auto name = session.GetOutputName(i, allocator); model_output_names_.push_back(name); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index d74fb1ba9ad32..680261c9c05cf 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -102,7 +102,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device size_t output_count = session_.GetOutputCount(); output_names_.resize(output_count); - Ort::Allocator a = Ort::Allocator::CreateDefault(); + Ort::Allocator a = Ort::Allocator::GetDefault(); for (size_t i = 0; i != output_count; ++i) { char* output_name = session_.GetOutputName(i, a); assert(output_name != nullptr); diff --git a/onnxruntime/test/server/unit_tests/converter_tests.cc b/onnxruntime/test/server/unit_tests/converter_tests.cc index 4b192dd0d08c7..d36f76c3ed2ce 100644 --- a/onnxruntime/test/server/unit_tests/converter_tests.cc +++ b/onnxruntime/test/server/unit_tests/converter_tests.cc @@ -304,7 +304,7 @@ TEST(MLValueToTensorProtoTests, UInt8ProtoRoundTrip) { tp.set_data_type(onnx::TensorProto_DataType_UINT8); Ort::Value ml_value{nullptr}; char buf[1000]; - auto allocator = Ort::Allocator::CreateDefault(); + auto allocator = Ort::Allocator::GetDefault(); auto info = allocator.GetInfo(); MemBuffer buffer((void*)&buf, tp.ByteSizeLong(), *info); onnxruntime::server::TensorProtoToMLValue(tp, buffer, ml_value); diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc index 372594fa8f16c..cfebc23f307d1 100644 --- a/onnxruntime/test/shared_lib/test_allocator.cc +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -20,7 +20,7 @@ TEST_F(CApiTest, allocation_info) { } TEST_F(CApiTest, DefaultAllocator) { - Ort::Allocator default_allocator = Ort::Allocator::CreateDefault(); + Ort::Allocator default_allocator = Ort::Allocator::GetDefault(); char* p = (char*)default_allocator.Alloc(100); ASSERT_NE(p, nullptr); memset(p, 0, 100); diff --git a/samples/c_cxx/imagenet/main.cc b/samples/c_cxx/imagenet/main.cc index 1077bcd40789f..1a088e10b8cf4 100644 --- a/samples/c_cxx/imagenet/main.cc +++ b/samples/c_cxx/imagenet/main.cc @@ -128,7 +128,7 @@ class Validator : public OutputCollector { CreateSession(); VerifyInputOutputCount(session_); OrtAllocator* ort_alloc; - ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&ort_alloc)); + ORT_THROW_ON_ERROR(OrtGetDefaultAllocator(&ort_alloc)); { char* t; ORT_THROW_ON_ERROR(OrtSessionGetInputName(session_, 0, ort_alloc, &t)); @@ -253,7 +253,7 @@ int main(int argc, ORTCHAR_T* argv[]) { try { ret = real_main(argc, argv); } catch (const std::exception& ex) { - fprintf(stderr, "%s\n", ex.what()); + fprintf(stderr, "%s\n", ex.what()); } #ifdef _WIN32 CoUninitialize(); From 1f0a904a0bf589b72199ffadc2ef7b7ad5fdc695 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Thu, 15 Aug 2019 16:06:23 -0700 Subject: [PATCH 4/8] updates... --- include/onnxruntime/core/session/onnxruntime_c_api.h | 2 ++ include/onnxruntime/core/session/onnxruntime_cxx_api.h | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 1c6c76f42ee33..95eeb664e9f03 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -441,6 +441,8 @@ ORT_API_STATUS(OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outpt ORT_API_STATUS(OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); ORT_API_STATUS(OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr, _Out_ const OrtAllocatorInfo** out); +// The returned pointer doesn't have to be freed. +// Always returns the same instance on every invocation. ORT_API_STATUS(OrtGetDefaultAllocator, _Outptr_ OrtAllocator** out); ORT_API(const char*, OrtGetVersionString); diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index d3605989bb762..24c19e1dee0cc 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -43,6 +43,7 @@ struct Exception : std::exception { #define ORT_DEFINE_RELEASE(NAME) \ inline void OrtRelease(Ort##NAME* ptr) { OrtRelease##NAME(ptr); } +ORT_DEFINE_RELEASE(Allocator); ORT_DEFINE_RELEASE(AllocatorInfo); ORT_DEFINE_RELEASE(CustomOpDomain); ORT_DEFINE_RELEASE(Env); @@ -229,11 +230,11 @@ struct Value : Base { TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; }; -struct Allocator : Base { +struct Allocator : Unowned> { static Allocator GetDefault(); explicit Allocator(nullptr_t) {} - explicit Allocator(OrtAllocator* p) : Base{p} {} + explicit Allocator(OrtAllocator* p) : Unowned>{p} {} void* Alloc(size_t size); void Free(void* p); From 003fbd429e97146985f4644e6f6b9669c80ba6c4 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Thu, 15 Aug 2019 16:06:23 -0700 Subject: [PATCH 5/8] updates... --- .../DisposableNamedOnnxValue.cs | 2 +- .../NativeMemoryAllocator.cs | 2 +- .../src/Microsoft.ML.OnnxRuntime/NativeMethods.cs | 2 +- .../CXX_Api_Sample.cpp | 2 +- .../C_Api_Sample.cpp | 2 +- .../InferenceTest.cs | 2 +- .../onnxruntime/core/session/onnxruntime_c_api.h | 5 +++-- .../onnxruntime/core/session/onnxruntime_cxx_api.h | 14 ++++++++++---- .../core/session/onnxruntime_cxx_inline.h | 4 ++-- onnxruntime/core/providers/cpu/symbols.txt | 2 +- .../core/session/default_cpu_allocator_c_api.cc | 2 +- onnxruntime/server/environment.cc | 2 +- onnxruntime/test/perftest/ort_test_session.cc | 2 +- .../test/server/unit_tests/converter_tests.cc | 2 +- onnxruntime/test/shared_lib/test_allocator.cc | 2 +- samples/c_cxx/imagenet/main.cc | 2 +- 16 files changed, 28 insertions(+), 21 deletions(-) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs index ebf3a7e7839e3..157bbbebae7c7 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs @@ -134,7 +134,7 @@ internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name, internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue) { IntPtr allocator = IntPtr.Zero; - NativeApiStatus.VerifySuccess(NativeMethods.OrtGetDefaultAllocator(out allocator)); + NativeApiStatus.VerifySuccess(NativeMethods.OrtGetAllocatorWithDefaultOptions(out allocator)); var ret = CreateFromOnnxValue(name, nativeOnnxValue, allocator); return (DisposableNamedOnnxValue)ret; } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs index a412316561dfa..50b961bf23eec 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs @@ -84,7 +84,7 @@ private static NativeMemoryAllocator GetDefaultCpuAllocator() IntPtr allocator = IntPtr.Zero; try { - IntPtr status = NativeMethods.OrtGetDefaultAllocator(out allocator); + IntPtr status = NativeMethods.OrtGetAllocatorWithDefaultOptions(out allocator); NativeApiStatus.VerifySuccess(status); } catch (Exception e) diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index ba42d2fbe7726..942ce608edb11 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -260,7 +260,7 @@ public enum MemoryType public static extern void OrtReleaseAllocatorInfo(IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo); [DllImport(nativeLib, CharSet = charSet)] - public static extern IntPtr /*(OrtStatus*)*/OrtGetDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator); + public static extern IntPtr /*(OrtStatus*)*/OrtGetAllocatorWithDefaultOptions(out IntPtr /*(OrtAllocator**)*/ allocator); /// /// Release any object allocated by an allocator diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp index 13c1f9a5a04f4..364ecdbd3c62f 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp @@ -43,7 +43,7 @@ int main(int argc, char* argv[]) { //************************************************************************* // print model input layer (node names, types, shape etc.) - Ort::Allocator allocator = Ort::Allocator::GetDefault(); + Ort::Allocator allocator = Ort::Allocator::GetWithDefaultOptions(); // print number of model input nodes size_t num_input_nodes = session.GetInputCount(); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp index 08b9129327583..bdc413715281c 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp @@ -59,7 +59,7 @@ int main(int argc, char* argv[]) { size_t num_input_nodes; OrtStatus* status; OrtAllocator* allocator; - CHECK_STATUS(OrtGetDefaultAllocator(&allocator)); + CHECK_STATUS(OrtGetAllocatorWithDefaultOptions(&allocator)); // print number of model input nodes status = OrtSessionGetInputCount(session, &num_input_nodes); diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index d8b3521e3ef6b..bd6d8abc4afb0 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -793,7 +793,7 @@ private void VerifyNativeMethodsExist() "OrtEnableMemPattern","OrtDisableMemPattern","OrtEnableCpuMemArena","OrtDisableCpuMemArena", "OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel", "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU","OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo", - "OrtGetDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo", + "OrtGetAllocatorWithDefaultOptions","OrtAllocatorFree","OrtAllocatorGetInfo", "OrtCreateTensorWithDataAsOrtValue","OrtGetTensorMutableData", "OrtReleaseAllocatorInfo", "OrtCastTypeInfoToTensorInfo","OrtGetTensorTypeAndShape","OrtGetTensorElementType","OrtGetDimensionsCount", "OrtGetDimensions","OrtGetTensorShapeElementCount","OrtReleaseValue"}; diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 1c6c76f42ee33..31ab249c5e1e2 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -151,7 +151,6 @@ ORT_RUNTIME_CLASS(TypeInfo); ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo); ORT_RUNTIME_CLASS(SessionOptions); ORT_RUNTIME_CLASS(CustomOpDomain); -ORT_RUNTIME_CLASS(Allocator); // When passing in an allocator to any ORT function, be sure that the allocator object // is not destroyed until the last allocated object using it is freed. @@ -441,7 +440,9 @@ ORT_API_STATUS(OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outpt ORT_API_STATUS(OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p); ORT_API_STATUS(OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr, _Out_ const OrtAllocatorInfo** out); -ORT_API_STATUS(OrtGetDefaultAllocator, _Outptr_ OrtAllocator** out); +// The returned pointer doesn't have to be freed. +// Always returns the same instance on every invocation. +ORT_API_STATUS(OrtGetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out); ORT_API(const char*, OrtGetVersionString); /** diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index d3605989bb762..61baac5336f93 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. +// Licensed bder the MIT License. // Summary: The Ort C++ API is a header only wrapper around the Ort C API. // @@ -229,16 +229,22 @@ struct Value : Base { TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; }; -struct Allocator : Base { - static Allocator GetDefault(); +struct Allocator { + static Allocator GetWithDefaultOptions(); explicit Allocator(nullptr_t) {} - explicit Allocator(OrtAllocator* p) : Base{p} {} + explicit Allocator(OrtAllocator* p) : p_{p} {} + + operator OrtAllocator*() { return p_; } + operator const OrtAllocator*() const { return p_; } void* Alloc(size_t size); void Free(void* p); const OrtAllocatorInfo* GetInfo() const; + + private: + OrtAllocator* p_{}; }; struct AllocatorInfo : Base { diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index f2739009535ba..de42bb9e5057b 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -39,9 +39,9 @@ struct TypeToTensorType { static constexpr ONNXTensorElementDataType t template <> struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; -inline Allocator Allocator::GetDefault() { +inline Allocator Allocator::GetWithDefaultOptions() { OrtAllocator* p; - ORT_THROW_ON_ERROR(OrtGetDefaultAllocator(&p)); + ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&p)); return Allocator(p); } diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt index b8505de14b3f1..1d2750e5d3d3e 100644 --- a/onnxruntime/core/providers/cpu/symbols.txt +++ b/onnxruntime/core/providers/cpu/symbols.txt @@ -12,7 +12,7 @@ OrtCompareAllocatorInfo OrtCreateAllocatorInfo OrtCreateCpuAllocatorInfo OrtCreateCustomOpDomain -OrtGetDefaultAllocator +OrtGetAllocatorWithDefaultOptions OrtCreateEnv OrtCreateEnvWithCustomLogger OrtCreateRunOptions diff --git a/onnxruntime/core/session/default_cpu_allocator_c_api.cc b/onnxruntime/core/session/default_cpu_allocator_c_api.cc index eb5f3a97b3d79..713a843232dfc 100644 --- a/onnxruntime/core/session/default_cpu_allocator_c_api.cc +++ b/onnxruntime/core/session/default_cpu_allocator_c_api.cc @@ -46,7 +46,7 @@ struct OrtDefaultAllocator : OrtAllocatorImpl { return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \ } -ORT_API_STATUS_IMPL(OrtGetDefaultAllocator, _Out_ OrtAllocator** out) { +ORT_API_STATUS_IMPL(OrtGetAllocatorWithDefaultOptions, _Out_ OrtAllocator** out) { API_IMPL_BEGIN static OrtDefaultAllocator ort_default_allocator; *out = &ort_default_allocator; diff --git a/onnxruntime/server/environment.cc b/onnxruntime/server/environment.cc index 9888ecba6e6ef..11c376ac3e979 100644 --- a/onnxruntime/server/environment.cc +++ b/onnxruntime/server/environment.cc @@ -48,7 +48,7 @@ void ServerEnvironment::InitializeModel(const std::string& model_path) { auto output_count = session.GetOutputCount(); - auto allocator = Ort::Allocator::GetDefault(); + auto allocator = Ort::Allocator::GetWithDefaultOptions(); for (size_t i = 0; i < output_count; i++) { auto name = session.GetOutputName(i, allocator); model_output_names_.push_back(name); diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 680261c9c05cf..41eea90a125c8 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -102,7 +102,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device size_t output_count = session_.GetOutputCount(); output_names_.resize(output_count); - Ort::Allocator a = Ort::Allocator::GetDefault(); + Ort::Allocator a = Ort::Allocator::GetWithDefaultOptions(); for (size_t i = 0; i != output_count; ++i) { char* output_name = session_.GetOutputName(i, a); assert(output_name != nullptr); diff --git a/onnxruntime/test/server/unit_tests/converter_tests.cc b/onnxruntime/test/server/unit_tests/converter_tests.cc index d36f76c3ed2ce..f53cb79f45642 100644 --- a/onnxruntime/test/server/unit_tests/converter_tests.cc +++ b/onnxruntime/test/server/unit_tests/converter_tests.cc @@ -304,7 +304,7 @@ TEST(MLValueToTensorProtoTests, UInt8ProtoRoundTrip) { tp.set_data_type(onnx::TensorProto_DataType_UINT8); Ort::Value ml_value{nullptr}; char buf[1000]; - auto allocator = Ort::Allocator::GetDefault(); + auto allocator = Ort::Allocator::GetWithDefaultOptions(); auto info = allocator.GetInfo(); MemBuffer buffer((void*)&buf, tp.ByteSizeLong(), *info); onnxruntime::server::TensorProtoToMLValue(tp, buffer, ml_value); diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc index cfebc23f307d1..af269b8d3d222 100644 --- a/onnxruntime/test/shared_lib/test_allocator.cc +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -20,7 +20,7 @@ TEST_F(CApiTest, allocation_info) { } TEST_F(CApiTest, DefaultAllocator) { - Ort::Allocator default_allocator = Ort::Allocator::GetDefault(); + Ort::Allocator default_allocator = Ort::Allocator::GetWithDefaultOptions(); char* p = (char*)default_allocator.Alloc(100); ASSERT_NE(p, nullptr); memset(p, 0, 100); diff --git a/samples/c_cxx/imagenet/main.cc b/samples/c_cxx/imagenet/main.cc index 89ce9cfcccd89..4a1b975c55540 100644 --- a/samples/c_cxx/imagenet/main.cc +++ b/samples/c_cxx/imagenet/main.cc @@ -128,7 +128,7 @@ class Validator : public OutputCollector { CreateSession(); VerifyInputOutputCount(session_); OrtAllocator* ort_alloc; - ORT_THROW_ON_ERROR(OrtGetDefaultAllocator(&ort_alloc)); + ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&ort_alloc)); { char* t; ORT_THROW_ON_ERROR(OrtSessionGetInputName(session_, 0, ort_alloc, &t)); From 12e2034a63e8106c097b130c652978844c6e7967 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Fri, 16 Aug 2019 17:53:08 -0700 Subject: [PATCH 6/8] PR comments --- .../CXX_Api_Sample.cpp | 2 +- .../onnxruntime/core/session/onnxruntime_cxx_api.h | 10 +++------- .../core/session/onnxruntime_cxx_inline.h | 12 +++++------- onnxruntime/test/perftest/ort_test_session.cc | 2 +- onnxruntime/test/shared_lib/test_allocator.cc | 2 +- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp index 364ecdbd3c62f..aee1479137f2e 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp @@ -43,7 +43,7 @@ int main(int argc, char* argv[]) { //************************************************************************* // print model input layer (node names, types, shape etc.) - Ort::Allocator allocator = Ort::Allocator::GetWithDefaultOptions(); + Ort::AllocatorWithDefaultOptions allocator; // print number of model input nodes size_t num_input_nodes = session.GetInputCount(); diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index bc800e79b0f67..18ec9fd94cebe 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -43,7 +43,6 @@ struct Exception : std::exception { #define ORT_DEFINE_RELEASE(NAME) \ inline void OrtRelease(Ort##NAME* ptr) { OrtRelease##NAME(ptr); } -ORT_DEFINE_RELEASE(Allocator); ORT_DEFINE_RELEASE(AllocatorInfo); ORT_DEFINE_RELEASE(CustomOpDomain); ORT_DEFINE_RELEASE(Env); @@ -93,7 +92,7 @@ struct Unowned : T { ~Unowned() { this->p_ = nullptr; } }; -struct Allocator; +struct AllocatorWithDefaultOptions; struct AllocatorInfo; struct Env; struct TypeInfo; @@ -230,11 +229,8 @@ struct Value : Base { TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const; }; -struct Allocator { - static Allocator GetWithDefaultOptions(); - - explicit Allocator(nullptr_t) {} - explicit Allocator(OrtAllocator* p) : p_{p} {} +struct AllocatorWithDefaultOptions { + AllocatorWithDefaultOptions(); operator OrtAllocator*() { return p_; } operator const OrtAllocator*() const { return p_; } diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index de42bb9e5057b..3670fdfe71bc6 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -39,23 +39,21 @@ struct TypeToTensorType { static constexpr ONNXTensorElementDataType t template <> struct TypeToTensorType { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; }; -inline Allocator Allocator::GetWithDefaultOptions() { - OrtAllocator* p; - ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&p)); - return Allocator(p); +inline AllocatorWithDefaultOptions::AllocatorWithDefaultOptions() { + ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&p_)); } -inline void* Allocator::Alloc(size_t size) { +inline void* AllocatorWithDefaultOptions::Alloc(size_t size) { void* out; ORT_THROW_ON_ERROR(OrtAllocatorAlloc(p_, size, &out)); return out; } -inline void Allocator::Free(void* p) { +inline void AllocatorWithDefaultOptions::Free(void* p) { ORT_THROW_ON_ERROR(OrtAllocatorFree(p_, p)); } -inline const OrtAllocatorInfo* Allocator::GetInfo() const { +inline const OrtAllocatorInfo* AllocatorWithDefaultOptions::GetInfo() const { const OrtAllocatorInfo* out; ORT_THROW_ON_ERROR(OrtAllocatorGetInfo(p_, &out)); return out; diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 41eea90a125c8..1b512394be070 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -102,7 +102,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device size_t output_count = session_.GetOutputCount(); output_names_.resize(output_count); - Ort::Allocator a = Ort::Allocator::GetWithDefaultOptions(); + Ort::AllocatorWithDefaultOptions a; for (size_t i = 0; i != output_count; ++i) { char* output_name = session_.GetOutputName(i, a); assert(output_name != nullptr); diff --git a/onnxruntime/test/shared_lib/test_allocator.cc b/onnxruntime/test/shared_lib/test_allocator.cc index af269b8d3d222..b20231f94d561 100644 --- a/onnxruntime/test/shared_lib/test_allocator.cc +++ b/onnxruntime/test/shared_lib/test_allocator.cc @@ -20,7 +20,7 @@ TEST_F(CApiTest, allocation_info) { } TEST_F(CApiTest, DefaultAllocator) { - Ort::Allocator default_allocator = Ort::Allocator::GetWithDefaultOptions(); + Ort::AllocatorWithDefaultOptions default_allocator; char* p = (char*)default_allocator.Alloc(100); ASSERT_NE(p, nullptr); memset(p, 0, 100); From b999b82d0fab248946353b50b4a1833e38c0dde0 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Fri, 16 Aug 2019 18:01:13 -0700 Subject: [PATCH 7/8] fix typo in license header --- include/onnxruntime/core/session/onnxruntime_cxx_api.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 18ec9fd94cebe..82263bb071a7a 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -1,5 +1,5 @@ // Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed bder the MIT License. +// Licensed under the MIT License. // Summary: The Ort C++ API is a header only wrapper around the Ort C API. // From 58863c4590fdb117a905aeeff7ed4397273507a2 Mon Sep 17 00:00:00 2001 From: Pranav Sharma Date: Tue, 20 Aug 2019 01:20:46 -0700 Subject: [PATCH 8/8] fix build --- onnxruntime/server/environment.cc | 2 +- onnxruntime/test/server/unit_tests/converter_tests.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/server/environment.cc b/onnxruntime/server/environment.cc index 11c376ac3e979..f4a87ee7c4821 100644 --- a/onnxruntime/server/environment.cc +++ b/onnxruntime/server/environment.cc @@ -48,7 +48,7 @@ void ServerEnvironment::InitializeModel(const std::string& model_path) { auto output_count = session.GetOutputCount(); - auto allocator = Ort::Allocator::GetWithDefaultOptions(); + Ort::AllocatorWithDefaultOptions allocator; for (size_t i = 0; i < output_count; i++) { auto name = session.GetOutputName(i, allocator); model_output_names_.push_back(name); diff --git a/onnxruntime/test/server/unit_tests/converter_tests.cc b/onnxruntime/test/server/unit_tests/converter_tests.cc index f53cb79f45642..d020584837e8e 100644 --- a/onnxruntime/test/server/unit_tests/converter_tests.cc +++ b/onnxruntime/test/server/unit_tests/converter_tests.cc @@ -304,7 +304,7 @@ TEST(MLValueToTensorProtoTests, UInt8ProtoRoundTrip) { tp.set_data_type(onnx::TensorProto_DataType_UINT8); Ort::Value ml_value{nullptr}; char buf[1000]; - auto allocator = Ort::Allocator::GetWithDefaultOptions(); + Ort::AllocatorWithDefaultOptions allocator; auto info = allocator.GetInfo(); MemBuffer buffer((void*)&buf, tp.ByteSizeLong(), *info); onnxruntime::server::TensorProtoToMLValue(tp, buffer, ml_value);