Don't create the default allocator every single time. Rename API accordingly. Expose Session/Run log severity levels.

csharp/src/Microsoft.ML.OnnxRuntime/DisposableNamedOnnxValue.cs

@@ -134,9 +134,8 @@ internal static DisposableNamedOnnxValue CreateTensorFromOnnxValue(string name,
         internal static DisposableNamedOnnxValue CreateFromOnnxValue(string name, IntPtr nativeOnnxValue)
         {
             IntPtr allocator = IntPtr.Zero;
-            NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateDefaultAllocator(out allocator));
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtGetAllocatorWithDefaultOptions(out allocator));
             var ret = CreateFromOnnxValue(name, nativeOnnxValue, allocator);
-            NativeMethods.OrtReleaseAllocator(allocator);
             return (DisposableNamedOnnxValue)ret;
         }
 

csharp/src/Microsoft.ML.OnnxRuntime/NativeMemoryAllocator.cs

@@ -77,22 +77,18 @@ protected override bool ReleaseHandle()
 
     internal class NativeMemoryAllocator : SafeHandle
     {
-        protected static readonly Lazy<NativeMemoryAllocator> _defaultInstance = new Lazy<NativeMemoryAllocator>(CreateDefaultCpuAllocator);
+        protected static readonly Lazy<NativeMemoryAllocator> _defaultInstance = new Lazy<NativeMemoryAllocator>(GetDefaultCpuAllocator);
 
-        private static NativeMemoryAllocator CreateDefaultCpuAllocator()
+        private static NativeMemoryAllocator GetDefaultCpuAllocator()
         {
             IntPtr allocator = IntPtr.Zero;
             try
             {
-                IntPtr status = NativeMethods.OrtCreateDefaultAllocator(out allocator);
+                IntPtr status = NativeMethods.OrtGetAllocatorWithDefaultOptions(out allocator);
                 NativeApiStatus.VerifySuccess(status);
             }
             catch (Exception e)
             {
-                if (allocator != IntPtr.Zero)
-                {
-                    Delete(allocator);
-                }
                 throw e;
             }
 
@@ -124,7 +120,7 @@ public override bool IsInvalid
             }
         }
 
-        internal IntPtr Handle  
+        internal IntPtr Handle
         {
             get
             {
@@ -138,15 +134,8 @@ protected NativeMemoryAllocator(IntPtr allocator)
             this.handle = allocator;
         }
 
-
-        protected static void Delete(IntPtr allocator)
-        {
-            NativeMethods.OrtReleaseAllocator(allocator);
-        }
-
         protected override bool ReleaseHandle()
         {
-            Delete(this.handle);
             return true;
         }
     }

csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs

@@ -157,6 +157,9 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSetSessionLogVerbosityLevel(IntPtr /* OrtSessionOptions* */ options, LogLevel sessionLogVerbosityLevel);
 
+        [DllImport(nativeLib, CharSet = charSet)]
+        public static extern IntPtr /*(OrtStatus*)*/ OrtSetSessionLogSeverityLevel(IntPtr /* OrtSessionOptions* */ options, LogLevel sessionLogSeverityLevel);
+
         [DllImport(nativeLib, CharSet = charSet)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSetSessionThreadPoolSize(IntPtr /* OrtSessionOptions* */ options, int sessionThreadPoolSize);
 
@@ -257,10 +260,7 @@ public enum MemoryType
         public static extern void OrtReleaseAllocatorInfo(IntPtr /*(OrtAllocatorInfo*)*/ allocatorInfo);
 
         [DllImport(nativeLib, CharSet = charSet)]
-        public static extern IntPtr /*(OrtStatus*)*/OrtCreateDefaultAllocator(out IntPtr /*(OrtAllocator**)*/ allocator);
-
-        [DllImport(nativeLib, CharSet = charSet)]
-        public static extern void OrtReleaseAllocator(IntPtr /*(OrtAllocator*)*/ allocator);
+        public static extern IntPtr /*(OrtStatus*)*/OrtGetAllocatorWithDefaultOptions(out IntPtr /*(OrtAllocator**)*/ allocator);
 
         /// <summary>
         /// Release any object allocated by an allocator

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/CXX_Api_Sample.cpp

@@ -43,7 +43,7 @@ int main(int argc, char* argv[]) {
 
   //*************************************************************************
   // print model input layer (node names, types, shape etc.)
-  Ort::Allocator allocator = Ort::Allocator::CreateDefault();
+  Ort::Allocator allocator = Ort::Allocator::GetWithDefaultOptions();
 
   // print number of model input nodes
   size_t num_input_nodes = session.GetInputCount();

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests.Capi/C_Api_Sample.cpp

@@ -59,7 +59,7 @@ int main(int argc, char* argv[]) {
   size_t num_input_nodes;
   OrtStatus* status;
   OrtAllocator* allocator;
-  CHECK_STATUS(OrtCreateDefaultAllocator(&allocator));
+  CHECK_STATUS(OrtGetAllocatorWithDefaultOptions(&allocator));
 
   // print number of model input nodes
   status = OrtSessionGetInputCount(session, &num_input_nodes);
@@ -97,7 +97,6 @@ int main(int argc, char* argv[]) {
 
     OrtReleaseTypeInfo(typeinfo);
   }
-  OrtReleaseAllocator(allocator);
 
   // Results should be...
   // Number of inputs = 1

csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs

@@ -792,11 +792,8 @@ private void VerifyNativeMethodsExist()
             "OrtEnableSequentialExecution","OrtDisableSequentialExecution","OrtEnableProfiling","OrtDisableProfiling",
             "OrtEnableMemPattern","OrtDisableMemPattern","OrtEnableCpuMemArena","OrtDisableCpuMemArena",
             "OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel",
-            "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU",
-            "OrtCreateRunOptions", "OrtReleaseRunOptions", "OrtRunOptionsSetRunLogVerbosityLevel", "OrtRunOptionsSetRunTag",
-            "OrtRunOptionsGetRunLogVerbosityLevel", "OrtRunOptionsGetRunTag","OrtRunOptionsEnableTerminate", "OrtRunOptionsDisableTerminate",
-            "OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo",
-            "OrtCreateDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo",
+            "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU","OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo",
+            "OrtGetAllocatorWithDefaultOptions","OrtAllocatorFree","OrtAllocatorGetInfo",
             "OrtCreateTensorWithDataAsOrtValue","OrtGetTensorMutableData", "OrtReleaseAllocatorInfo",
             "OrtCastTypeInfoToTensorInfo","OrtGetTensorTypeAndShape","OrtGetTensorElementType","OrtGetDimensionsCount",
             "OrtGetDimensions","OrtGetTensorShapeElementCount","OrtReleaseValue"};

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -151,7 +151,6 @@ ORT_RUNTIME_CLASS(TypeInfo);
 ORT_RUNTIME_CLASS(TensorTypeAndShapeInfo);
 ORT_RUNTIME_CLASS(SessionOptions);
 ORT_RUNTIME_CLASS(CustomOpDomain);
-ORT_RUNTIME_CLASS(Allocator);
 
 // When passing in an allocator to any ORT function, be sure that the allocator object
 // is not destroyed until the last allocated object using it is freed.
@@ -202,7 +201,7 @@ ORT_API_STATUS(OrtRun, _Inout_ OrtSession* sess,
 ORT_API_STATUS(OrtCreateSessionOptions, _Outptr_ OrtSessionOptions** options);
 
 // Set filepath to save optimized model after graph level transformations.
-ORT_API_STATUS(OrtSetOptimizedModelFilePath, _In_ OrtSessionOptions* options, _In_ const ORTCHAR_T* optimized_model_filepath);
+ORT_API_STATUS(OrtSetOptimizedModelFilePath, _Inout_ OrtSessionOptions* options, _In_ const ORTCHAR_T* optimized_model_filepath);
 
 // create a copy of an existing OrtSessionOptions
 ORT_API_STATUS(OrtCloneSessionOptions, _In_ const OrtSessionOptions* in_options, _Outptr_ OrtSessionOptions** out_options);
@@ -232,6 +231,7 @@ ORT_API_STATUS(OrtSetSessionLogId, _Inout_ OrtSessionOptions* options, const cha
 
 // < applies to session load, initialization, etc
 ORT_API_STATUS(OrtSetSessionLogVerbosityLevel, _Inout_ OrtSessionOptions* options, int session_log_verbosity_level);
+ORT_API_STATUS(OrtSetSessionLogSeverityLevel, _Inout_ OrtSessionOptions* options, int session_log_severity_level);
 
 // Set Graph optimization level.
 // TODO Add documentation about which optimizations are enabled for each value.
@@ -285,9 +285,11 @@ ORT_API_STATUS(OrtSessionGetOutputName, _In_ const OrtSession* sess, size_t inde
 ORT_API_STATUS(OrtCreateRunOptions, _Outptr_ OrtRunOptions** out);
 
 ORT_API_STATUS(OrtRunOptionsSetRunLogVerbosityLevel, _Inout_ OrtRunOptions* options, int value);
+ORT_API_STATUS(OrtRunOptionsSetRunLogSeverityLevel, _Inout_ OrtRunOptions* options, int value);
 ORT_API_STATUS(OrtRunOptionsSetRunTag, _In_ OrtRunOptions*, _In_ const char* run_tag);
 
 ORT_API_STATUS(OrtRunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
+ORT_API_STATUS(OrtRunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, _Out_ int* out);
 ORT_API_STATUS(OrtRunOptionsGetRunTag, _In_ const OrtRunOptions*, _Out_ const char** out);
 
 // Set a flag so that any running OrtRun* calls that are using this instance of OrtRunOptions
@@ -438,7 +440,9 @@ ORT_API_STATUS(OrtAllocatorAlloc, _Inout_ OrtAllocator* ptr, size_t size, _Outpt
 ORT_API_STATUS(OrtAllocatorFree, _Inout_ OrtAllocator* ptr, void* p);
 ORT_API_STATUS(OrtAllocatorGetInfo, _In_ const OrtAllocator* ptr, _Out_ const OrtAllocatorInfo** out);
 
-ORT_API_STATUS(OrtCreateDefaultAllocator, _Outptr_ OrtAllocator** out);
+// The returned pointer doesn't have to be freed.
+// Always returns the same instance on every invocation.
+ORT_API_STATUS(OrtGetAllocatorWithDefaultOptions, _Outptr_ OrtAllocator** out);
 
 ORT_API(const char*, OrtGetVersionString);
 /**

include/onnxruntime/core/session/onnxruntime_cxx_api.h

@@ -1,5 +1,5 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
+// Licensed bder the MIT License.
 
 // Summary: The Ort C++ API is a header only wrapper around the Ort C API.
 //
@@ -43,7 +43,6 @@ struct Exception : std::exception {
 #define ORT_DEFINE_RELEASE(NAME) \
   inline void OrtRelease(Ort##NAME* ptr) { OrtRelease##NAME(ptr); }
 
-ORT_DEFINE_RELEASE(Allocator);
 ORT_DEFINE_RELEASE(AllocatorInfo);
 ORT_DEFINE_RELEASE(CustomOpDomain);
 ORT_DEFINE_RELEASE(Env);
@@ -120,6 +119,9 @@ struct RunOptions : Base<OrtRunOptions> {
   RunOptions& SetRunLogVerbosityLevel(int);
   int GetRunLogVerbosityLevel() const;
 
+  RunOptions& SetRunLogSeverityLevel(int);
+  int GetRunLogSeverityLevel() const;
+
   RunOptions& SetRunTag(const char* run_tag);
   const char* GetRunTag() const;
 
@@ -227,16 +229,22 @@ struct Value : Base<OrtValue> {
   TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const;
 };
 
-struct Allocator : Base<OrtAllocator> {
-  static Allocator CreateDefault();
+struct Allocator {
+  static Allocator GetWithDefaultOptions();
 
   explicit Allocator(nullptr_t) {}
-  explicit Allocator(OrtAllocator* p) : Base<OrtAllocator>{p} {}
+  explicit Allocator(OrtAllocator* p) : p_{p} {}
+
+  operator OrtAllocator*() { return p_; }
+  operator const OrtAllocator*() const { return p_; }
 
   void* Alloc(size_t size);
   void Free(void* p);
 
   const OrtAllocatorInfo* GetInfo() const;
+
+ private:
+  OrtAllocator* p_{};
 };
 
 struct AllocatorInfo : Base<OrtAllocatorInfo> {

include/onnxruntime/core/session/onnxruntime_cxx_inline.h

@@ -39,9 +39,9 @@ struct TypeToTensorType<uint32_t> { static constexpr ONNXTensorElementDataType t
 template <>
 struct TypeToTensorType<uint64_t> { static constexpr ONNXTensorElementDataType type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64; };
 
-inline Allocator Allocator::CreateDefault() {
+inline Allocator Allocator::GetWithDefaultOptions() {
   OrtAllocator* p;
-  ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&p));
+  ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&p));
   return Allocator(p);
 }
 
@@ -96,6 +96,11 @@ inline RunOptions& RunOptions::SetRunLogVerbosityLevel(int level) {
   return *this;
 }
 
+inline RunOptions& RunOptions::SetRunLogSeverityLevel(int level) {
+  ORT_THROW_ON_ERROR(OrtRunOptionsSetRunLogSeverityLevel(p_, level));
+  return *this;
+}
+
 inline int RunOptions::GetRunLogVerbosityLevel() const {
   int out;
   ORT_THROW_ON_ERROR(OrtRunOptionsGetRunLogVerbosityLevel(p_, &out));

onnxruntime/core/framework/run_options.cc

@@ -17,6 +17,11 @@ ORT_API_STATUS_IMPL(OrtRunOptionsSetRunLogVerbosityLevel, _In_ OrtRunOptions* op
   return nullptr;
 }
 
+ORT_API_STATUS_IMPL(OrtRunOptionsSetRunLogSeverityLevel, _In_ OrtRunOptions* options, int value) {
+  options->run_log_severity_level = value;
+  return nullptr;
+}
+
 ORT_API_STATUS_IMPL(OrtRunOptionsSetRunTag, _In_ OrtRunOptions* options, _In_ const char* run_tag) {
   if (run_tag)
     options->run_tag = run_tag;
@@ -28,6 +33,11 @@ ORT_API_STATUS_IMPL(OrtRunOptionsGetRunLogVerbosityLevel, _In_ const OrtRunOptio
   return nullptr;
 }
 
+ORT_API_STATUS_IMPL(OrtRunOptionsGetRunLogSeverityLevel, _In_ const OrtRunOptions* options, int* out) {
+  *out = options->run_log_severity_level;
+  return nullptr;
+}
+
 ORT_API_STATUS_IMPL(OrtRunOptionsGetRunTag, _In_ const OrtRunOptions* options, const char** out) {
   *out = options->run_tag.c_str();
   return nullptr;

onnxruntime/core/providers/cpu/symbols.txt

@@ -12,7 +12,7 @@ OrtCompareAllocatorInfo
 OrtCreateAllocatorInfo
 OrtCreateCpuAllocatorInfo
 OrtCreateCustomOpDomain
-OrtCreateDefaultAllocator
+OrtGetAllocatorWithDefaultOptions
 OrtCreateEnv
 OrtCreateEnvWithCustomLogger
 OrtCreateRunOptions
@@ -51,7 +51,6 @@ OrtGetValueType
 OrtGetVersionString
 OrtIsTensor
 OrtGetOnnxTypeFromTypeInfo
-OrtReleaseAllocator
 OrtReleaseAllocatorInfo
 OrtReleaseCustomOpDomain
 OrtReleaseEnv
@@ -66,6 +65,7 @@ OrtRun
 OrtRunOptionsGetRunLogVerbosityLevel
 OrtRunOptionsGetRunTag
 OrtRunOptionsSetRunLogVerbosityLevel
+OrtRunOptionsSetRunLogSeverityLevel
 OrtRunOptionsSetRunTag
 OrtRunOptionsEnableTerminate
 OrtRunOptionsDisableTerminate
@@ -80,6 +80,7 @@ OrtSetDimensions
 OrtSetSessionGraphOptimizationLevel
 OrtSetSessionLogId
 OrtSetSessionLogVerbosityLevel
+OrtSetSessionLogSeverityLevel
 OrtSetOptimizedModelFilePath
 OrtSetSessionThreadPoolSize
 OrtSetTensorElementType

onnxruntime/core/session/abi_session_options.cc

@@ -100,6 +100,11 @@ ORT_API_STATUS_IMPL(OrtSetSessionLogVerbosityLevel, _In_ OrtSessionOptions* opti
   return nullptr;
 }
 
+ORT_API_STATUS_IMPL(OrtSetSessionLogSeverityLevel, _In_ OrtSessionOptions* options, int session_log_severity_level) {
+  options->value.session_log_severity_level = session_log_severity_level;
+  return nullptr;
+}
+
 // Set Graph optimization level.
 ORT_API_STATUS_IMPL(OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions* options,
                     GraphOptimizationLevel graph_optimization_level) {

onnxruntime/core/session/default_cpu_allocator_c_api.cc

@@ -46,13 +46,10 @@ struct OrtDefaultAllocator : OrtAllocatorImpl {
     return OrtCreateStatus(ORT_RUNTIME_EXCEPTION, ex.what()); \
   }
 
-ORT_API_STATUS_IMPL(OrtCreateDefaultAllocator, _Out_ OrtAllocator** out) {
+ORT_API_STATUS_IMPL(OrtGetAllocatorWithDefaultOptions, _Out_ OrtAllocator** out) {
   API_IMPL_BEGIN
-  *out = new OrtDefaultAllocator();
+  static OrtDefaultAllocator ort_default_allocator;
+  *out = &ort_default_allocator;
   return nullptr;
   API_IMPL_END
 }
-
-ORT_API(void, OrtReleaseAllocator, _In_ OrtAllocator* allocator) {
-  delete static_cast<OrtAllocatorImpl*>(allocator);
-}

onnxruntime/server/environment.cc

@@ -48,7 +48,7 @@ void ServerEnvironment::InitializeModel(const std::string& model_path) {
 
   auto output_count = session.GetOutputCount();
 
-  auto allocator = Ort::Allocator::CreateDefault();
+  auto allocator = Ort::Allocator::GetWithDefaultOptions();
   for (size_t i = 0; i < output_count; i++) {
     auto name = session.GetOutputName(i, allocator);
     model_output_names_.push_back(name);

onnxruntime/test/perftest/ort_test_session.cc

@@ -102,7 +102,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
 
   size_t output_count = session_.GetOutputCount();
   output_names_.resize(output_count);
-  Ort::Allocator a = Ort::Allocator::CreateDefault();
+  Ort::Allocator a = Ort::Allocator::GetWithDefaultOptions();
   for (size_t i = 0; i != output_count; ++i) {
     char* output_name = session_.GetOutputName(i, a);
     assert(output_name != nullptr);

onnxruntime/test/server/unit_tests/converter_tests.cc

@@ -304,7 +304,7 @@ TEST(MLValueToTensorProtoTests, UInt8ProtoRoundTrip) {
   tp.set_data_type(onnx::TensorProto_DataType_UINT8);
   Ort::Value ml_value{nullptr};
   char buf[1000];
-  auto allocator = Ort::Allocator::CreateDefault();
+  auto allocator = Ort::Allocator::GetWithDefaultOptions();
   auto info = allocator.GetInfo();
   MemBuffer buffer((void*)&buf, tp.ByteSizeLong(), *info);
   onnxruntime::server::TensorProtoToMLValue(tp, buffer, ml_value);

onnxruntime/test/shared_lib/test_allocator.cc

@@ -20,7 +20,7 @@ TEST_F(CApiTest, allocation_info) {
 }
 
 TEST_F(CApiTest, DefaultAllocator) {
-  Ort::Allocator default_allocator = Ort::Allocator::CreateDefault();
+  Ort::Allocator default_allocator = Ort::Allocator::GetWithDefaultOptions();
   char* p = (char*)default_allocator.Alloc(100);
   ASSERT_NE(p, nullptr);
   memset(p, 0, 100);

samples/c_cxx/imagenet/main.cc

@@ -128,7 +128,7 @@ class Validator : public OutputCollector<TCharString> {
     CreateSession();
     VerifyInputOutputCount(session_);
     OrtAllocator* ort_alloc;
-    ORT_THROW_ON_ERROR(OrtCreateDefaultAllocator(&ort_alloc));
+    ORT_THROW_ON_ERROR(OrtGetAllocatorWithDefaultOptions(&ort_alloc));
     {
       char* t;
       ORT_THROW_ON_ERROR(OrtSessionGetInputName(session_, 0, ort_alloc, &t));
@@ -139,7 +139,6 @@ class Validator : public OutputCollector<TCharString> {
       OrtAllocatorFree(ort_alloc, t);
     }
 
-    OrtReleaseAllocator(ort_alloc);
     OrtTypeInfo* info;
     ORT_THROW_ON_ERROR(OrtSessionGetInputTypeInfo(session_, 0, &info));
     const OrtTensorTypeAndShapeInfo* tensor_info;
@@ -253,7 +252,7 @@ int main(int argc, ORTCHAR_T* argv[]) {
   try {
     ret = real_main(argc, argv);
   } catch (const std::exception& ex) {
-    fprintf(stderr, "%s\n", ex.what());    
+    fprintf(stderr, "%s\n", ex.what());
   }
 #ifdef _WIN32
   CoUninitialize();